{
  "source": "Saved MLflow benchmark report",
  "mlflow_run_id": "404403a1946146d889a7e2e7d25d4aac",
  "completed_at": "2026-10-02T12:39:03.102370+00:00",
  "models": [
    "gpt-5.6-sol",
    "deepseek-v4-flash",
    "claude-sonnet-5.5"
  ],
  "price_snapshot": {
    "captured_at": "2026-10-02T12:18:13.493Z",
    "source": "ci_catalog",
    "currency": "USD",
    "models": {
      "gpt-5.6-sol": {
        "input_per_million": 2,
        "output_per_million": 10
      },
      "deepseek-v4-flash": {
        "input_per_million": 0.09,
        "output_per_million": 0.36
      },
      "claude-sonnet-5.5": {
        "input_per_million": 1.4,
        "output_per_million": 7
      }
    }
  },
  "arms": [
    {
      "model": "gpt-5.6-sol",
      "variant": "baseline",
      "quality": {
        "reference": {
          "n": 0,
          "passed": 0,
          "evaluated": 0,
          "evaluation_errors": 0,
          "failed": 0,
          "pass_rate": null,
          "interval_95": null,
          "mean_cases": 0,
          "mean_score": null,
          "mean_interval_95": null
        },
        "rubric": {
          "n": 20,
          "passed": 20,
          "evaluated": 20,
          "evaluation_errors": 0,
          "failed": 0,
          "pass_rate": 1.0,
          "interval_95": [
            0.8388748419471806,
            1
          ],
          "mean_cases": 20,
          "mean_score": 1.0,
          "mean_interval_95": [
            1.0,
            1.0
          ],
          "criteria": [
            {
              "criterion": "3 keywords in total.",
              "method": "deterministic",
              "n": 20,
              "evaluated": 20,
              "passed": 20,
              "evaluation_errors": 0,
              "pass_rate": 1.0,
              "interval_95": [
                0.8388748419471806,
                1
              ]
            },
            {
              "criterion": "Each keyword must contain less than or equal to 5 words.",
              "method": "deterministic",
              "n": 20,
              "evaluated": 20,
              "passed": 20,
              "evaluation_errors": 0,
              "pass_rate": 1.0,
              "interval_95": [
                0.8388748419471806,
                1
              ]
            },
            {
              "criterion": "Each keyword must be relevant to the healthcare domain.",
              "method": "llm_judge",
              "n": 20,
              "evaluated": 20,
              "passed": 20,
              "evaluation_errors": 0,
              "pass_rate": 1.0,
              "interval_95": [
                0.8388748419471806,
                1
              ]
            }
          ]
        }
      },
      "usage": {
        "measured_cases": 20,
        "total_cases": 20,
        "avg_input": 78.25,
        "avg_output": 68.7
      },
      "estimated_cost_per_call": 0.0008435,
      "errors": 0
    },
    {
      "model": "gpt-5.6-sol",
      "variant": "optimized",
      "quality": {
        "reference": {
          "n": 0,
          "passed": 0,
          "evaluated": 0,
          "evaluation_errors": 0,
          "failed": 0,
          "pass_rate": null,
          "interval_95": null,
          "mean_cases": 0,
          "mean_score": null,
          "mean_interval_95": null
        },
        "rubric": {
          "n": 20,
          "passed": 20,
          "evaluated": 20,
          "evaluation_errors": 0,
          "failed": 0,
          "pass_rate": 1.0,
          "interval_95": [
            0.8388748419471806,
            1
          ],
          "mean_cases": 20,
          "mean_score": 1.0,
          "mean_interval_95": [
            1.0,
            1.0
          ],
          "criteria": [
            {
              "criterion": "3 keywords in total.",
              "method": "deterministic",
              "n": 20,
              "evaluated": 20,
              "passed": 20,
              "evaluation_errors": 0,
              "pass_rate": 1.0,
              "interval_95": [
                0.8388748419471806,
                1
              ]
            },
            {
              "criterion": "Each keyword must contain less than or equal to 5 words.",
              "method": "deterministic",
              "n": 20,
              "evaluated": 20,
              "passed": 20,
              "evaluation_errors": 0,
              "pass_rate": 1.0,
              "interval_95": [
                0.8388748419471806,
                1
              ]
            },
            {
              "criterion": "Each keyword must be relevant to the healthcare domain.",
              "method": "llm_judge",
              "n": 20,
              "evaluated": 20,
              "passed": 20,
              "evaluation_errors": 0,
              "pass_rate": 1.0,
              "interval_95": [
                0.8388748419471806,
                1
              ]
            }
          ]
        }
      },
      "usage": {
        "measured_cases": 20,
        "total_cases": 20,
        "avg_input": 313.5,
        "avg_output": 53.45
      },
      "estimated_cost_per_call": 0.0011615,
      "errors": 0
    },
    {
      "model": "deepseek-v4-flash",
      "variant": "baseline",
      "quality": {
        "reference": {
          "n": 0,
          "passed": 0,
          "evaluated": 0,
          "evaluation_errors": 0,
          "failed": 0,
          "pass_rate": null,
          "interval_95": null,
          "mean_cases": 0,
          "mean_score": null,
          "mean_interval_95": null
        },
        "rubric": {
          "n": 20,
          "passed": 19,
          "evaluated": 20,
          "evaluation_errors": 0,
          "failed": 1,
          "pass_rate": 0.95,
          "interval_95": [
            0.763868806553258,
            0.9911185511992047
          ],
          "mean_cases": 20,
          "mean_score": 0.9666666666666666,
          "mean_interval_95": [
            0.9,
            1.0
          ],
          "criteria": [
            {
              "criterion": "3 keywords in total.",
              "method": "deterministic",
              "n": 20,
              "evaluated": 20,
              "passed": 19,
              "evaluation_errors": 0,
              "pass_rate": 0.95,
              "interval_95": [
                0.763868806553258,
                0.9911185511992047
              ]
            },
            {
              "criterion": "Each keyword must contain less than or equal to 5 words.",
              "method": "deterministic",
              "n": 20,
              "evaluated": 20,
              "passed": 19,
              "evaluation_errors": 0,
              "pass_rate": 0.95,
              "interval_95": [
                0.763868806553258,
                0.9911185511992047
              ]
            },
            {
              "criterion": "Each keyword must be relevant to the healthcare domain.",
              "method": "llm_judge",
              "n": 20,
              "evaluated": 20,
              "passed": 20,
              "evaluation_errors": 0,
              "pass_rate": 1.0,
              "interval_95": [
                0.8388748419471806,
                1
              ]
            }
          ]
        }
      },
      "usage": {
        "measured_cases": 20,
        "total_cases": 20,
        "avg_input": 72.25,
        "avg_output": 166.35
      },
      "estimated_cost_per_call": 6.63885e-05,
      "errors": 0
    },
    {
      "model": "deepseek-v4-flash",
      "variant": "optimized",
      "quality": {
        "reference": {
          "n": 0,
          "passed": 0,
          "evaluated": 0,
          "evaluation_errors": 0,
          "failed": 0,
          "pass_rate": null,
          "interval_95": null,
          "mean_cases": 0,
          "mean_score": null,
          "mean_interval_95": null
        },
        "rubric": {
          "n": 20,
          "passed": 20,
          "evaluated": 20,
          "evaluation_errors": 0,
          "failed": 0,
          "pass_rate": 1.0,
          "interval_95": [
            0.8388748419471806,
            1
          ],
          "mean_cases": 20,
          "mean_score": 1.0,
          "mean_interval_95": [
            1.0,
            1.0
          ],
          "criteria": [
            {
              "criterion": "3 keywords in total.",
              "method": "deterministic",
              "n": 20,
              "evaluated": 20,
              "passed": 20,
              "evaluation_errors": 0,
              "pass_rate": 1.0,
              "interval_95": [
                0.8388748419471806,
                1
              ]
            },
            {
              "criterion": "Each keyword must contain less than or equal to 5 words.",
              "method": "deterministic",
              "n": 20,
              "evaluated": 20,
              "passed": 20,
              "evaluation_errors": 0,
              "pass_rate": 1.0,
              "interval_95": [
                0.8388748419471806,
                1
              ]
            },
            {
              "criterion": "Each keyword must be relevant to the healthcare domain.",
              "method": "llm_judge",
              "n": 20,
              "evaluated": 20,
              "passed": 20,
              "evaluation_errors": 0,
              "pass_rate": 1.0,
              "interval_95": [
                0.8388748419471806,
                1
              ]
            }
          ]
        }
      },
      "usage": {
        "measured_cases": 20,
        "total_cases": 20,
        "avg_input": 310.5,
        "avg_output": 152.55
      },
      "estimated_cost_per_call": 8.2863e-05,
      "errors": 0
    },
    {
      "model": "claude-sonnet-5.5",
      "variant": "baseline",
      "quality": {
        "reference": {
          "n": 0,
          "passed": 0,
          "evaluated": 0,
          "evaluation_errors": 0,
          "failed": 0,
          "pass_rate": null,
          "interval_95": null,
          "mean_cases": 0,
          "mean_score": null,
          "mean_interval_95": null
        },
        "rubric": {
          "n": 20,
          "passed": 0,
          "evaluated": 20,
          "evaluation_errors": 0,
          "failed": 20,
          "pass_rate": 0.0,
          "interval_95": [
            0,
            0.16112515805281938
          ],
          "mean_cases": 20,
          "mean_score": 0.5333333333333333,
          "mean_interval_95": [
            0.4,
            0.6333333333333333
          ],
          "criteria": [
            {
              "criterion": "3 keywords in total.",
              "method": "deterministic",
              "n": 20,
              "evaluated": 20,
              "passed": 16,
              "evaluation_errors": 0,
              "pass_rate": 0.8,
              "interval_95": [
                0.5839825677481064,
                0.919342337420202
              ]
            },
            {
              "criterion": "Each keyword must contain less than or equal to 5 words.",
              "method": "deterministic",
              "n": 20,
              "evaluated": 20,
              "passed": 0,
              "evaluation_errors": 0,
              "pass_rate": 0.0,
              "interval_95": [
                0,
                0.16112515805281938
              ]
            },
            {
              "criterion": "Each keyword must be relevant to the healthcare domain.",
              "method": "llm_judge",
              "n": 20,
              "evaluated": 20,
              "passed": 16,
              "evaluation_errors": 0,
              "pass_rate": 0.8,
              "interval_95": [
                0.5839825677481064,
                0.919342337420202
              ]
            }
          ]
        }
      },
      "usage": {
        "measured_cases": 20,
        "total_cases": 20,
        "avg_input": 133.65,
        "avg_output": 261.5
      },
      "estimated_cost_per_call": 0.00201761,
      "errors": 0
    },
    {
      "model": "claude-sonnet-5.5",
      "variant": "optimized",
      "quality": {
        "reference": {
          "n": 0,
          "passed": 0,
          "evaluated": 0,
          "evaluation_errors": 0,
          "failed": 0,
          "pass_rate": null,
          "interval_95": null,
          "mean_cases": 0,
          "mean_score": null,
          "mean_interval_95": null
        },
        "rubric": {
          "n": 20,
          "passed": 20,
          "evaluated": 20,
          "evaluation_errors": 0,
          "failed": 0,
          "pass_rate": 1.0,
          "interval_95": [
            0.8388748419471806,
            1
          ],
          "mean_cases": 20,
          "mean_score": 1.0,
          "mean_interval_95": [
            1.0,
            1.0
          ],
          "criteria": [
            {
              "criterion": "3 keywords in total.",
              "method": "deterministic",
              "n": 20,
              "evaluated": 20,
              "passed": 20,
              "evaluation_errors": 0,
              "pass_rate": 1.0,
              "interval_95": [
                0.8388748419471806,
                1
              ]
            },
            {
              "criterion": "Each keyword must contain less than or equal to 5 words.",
              "method": "deterministic",
              "n": 20,
              "evaluated": 20,
              "passed": 20,
              "evaluation_errors": 0,
              "pass_rate": 1.0,
              "interval_95": [
                0.8388748419471806,
                1
              ]
            },
            {
              "criterion": "Each keyword must be relevant to the healthcare domain.",
              "method": "llm_judge",
              "n": 20,
              "evaluated": 20,
              "passed": 20,
              "evaluation_errors": 0,
              "pass_rate": 1.0,
              "interval_95": [
                0.8388748419471806,
                1
              ]
            }
          ]
        }
      },
      "usage": {
        "measured_cases": 20,
        "total_cases": 20,
        "avg_input": 558.3,
        "avg_output": 39.65
      },
      "estimated_cost_per_call": 0.00105917,
      "errors": 0
    }
  ]
}