{
  "edition": "2026-11",
  "generated_at": "2026-11-16T02:26:01.686Z",
  "methodology_version": "1.0",
  "task_suite_sha256": "c47a513b0b5bb44bf5c28e6d17d136e3e7680e24b0fa1be524a335932ce73c81",
  "task_count": 40,
  "samples": 8,
  "temperature": 0.7,
  "max_tokens": 400,
  "dry_run": false,
  "models": [
    {
      "model": "google/gemma-4-31b-it",
      "pass_at_1": {
        "rate": 0.9625,
        "ci": [
          0.9356,
          0.9784
        ],
        "n": 320
      },
      "curve": [
        {
          "k": 1,
          "pass_at_k": 0.9625,
          "pass_at_k_ci": [
            0.9,
            1
          ],
          "pass_hat_k": 0.9625,
          "pass_hat_k_ci": [
            0.9,
            1
          ]
        },
        {
          "k": 2,
          "pass_at_k": 0.9696,
          "pass_at_k_ci": [
            0.9143,
            1
          ],
          "pass_hat_k": 0.9554,
          "pass_hat_k_ci": [
            0.8857,
            1
          ]
        },
        {
          "k": 3,
          "pass_at_k": 0.9732,
          "pass_at_k_ci": [
            0.9214,
            1
          ],
          "pass_hat_k": 0.9518,
          "pass_hat_k_ci": [
            0.8786,
            1
          ]
        },
        {
          "k": 4,
          "pass_at_k": 0.9746,
          "pass_at_k_ci": [
            0.9243,
            1
          ],
          "pass_hat_k": 0.9504,
          "pass_hat_k_ci": [
            0.8761,
            1
          ]
        },
        {
          "k": 5,
          "pass_at_k": 0.975,
          "pass_at_k_ci": [
            0.925,
            1
          ],
          "pass_hat_k": 0.95,
          "pass_hat_k_ci": [
            0.875,
            1
          ]
        },
        {
          "k": 6,
          "pass_at_k": 0.975,
          "pass_at_k_ci": [
            0.925,
            1
          ],
          "pass_hat_k": 0.95,
          "pass_hat_k_ci": [
            0.875,
            1
          ]
        },
        {
          "k": 7,
          "pass_at_k": 0.975,
          "pass_at_k_ci": [
            0.925,
            1
          ],
          "pass_hat_k": 0.95,
          "pass_hat_k_ci": [
            0.875,
            1
          ]
        },
        {
          "k": 8,
          "pass_at_k": 0.975,
          "pass_at_k_ci": [
            0.925,
            1
          ],
          "pass_hat_k": 0.95,
          "pass_hat_k_ci": [
            0.875,
            1
          ]
        }
      ],
      "by_category": {
        "reasoning": {
          "rate": 1,
          "ci": [
            0.9542,
            1
          ],
          "n": 80
        },
        "extraction": {
          "rate": 1,
          "ci": [
            0.9542,
            1
          ],
          "n": 80
        },
        "instruction": {
          "rate": 0.85,
          "ci": [
            0.7559,
            0.9121
          ],
          "n": 80
        },
        "json": {
          "rate": 1,
          "ci": [
            0.9542,
            1
          ],
          "n": 80
        }
      },
      "flaky": [
        "instruction-09"
      ],
      "tasks": [
        {
          "id": "reasoning-01",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-02",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-03",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-04",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-05",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-06",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-07",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-08",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-09",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-10",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-01",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-02",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-03",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-04",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-05",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-06",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-07",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-08",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-09",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-10",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-01",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-02",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-03",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-04",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-05",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-06",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-07",
          "category": "instruction",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "instruction-08",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-09",
          "category": "instruction",
          "n": 8,
          "c": 4,
          "errors": 0
        },
        {
          "id": "instruction-10",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-01",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-02",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-03",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-04",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-05",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-06",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-07",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-08",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-09",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-10",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        }
      ],
      "calls": 320,
      "errors": 0,
      "latency_ms": {
        "p50": 1193,
        "p95": 3234,
        "mean": 1459
      },
      "tokens": {
        "input": 26656,
        "output": 6772
      },
      "cost_usd": 0.006441,
      "cost_coverage": 1
    },
    {
      "model": "anthropic/claude-sonnet-5.5",
      "pass_at_1": {
        "rate": 0.9563,
        "ci": [
          0.9279,
          0.9738
        ],
        "n": 320
      },
      "curve": [
        {
          "k": 1,
          "pass_at_k": 0.9563,
          "pass_at_k_ci": [
            0.8906,
            1
          ],
          "pass_hat_k": 0.9563,
          "pass_hat_k_ci": [
            0.8938,
            1
          ]
        },
        {
          "k": 2,
          "pass_at_k": 0.9661,
          "pass_at_k_ci": [
            0.9071,
            1
          ],
          "pass_hat_k": 0.9464,
          "pass_hat_k_ci": [
            0.8741,
            1
          ]
        },
        {
          "k": 3,
          "pass_at_k": 0.9705,
          "pass_at_k_ci": [
            0.9161,
            1
          ],
          "pass_hat_k": 0.9411,
          "pass_hat_k_ci": [
            0.867,
            1
          ]
        },
        {
          "k": 4,
          "pass_at_k": 0.9732,
          "pass_at_k_ci": [
            0.9196,
            1
          ],
          "pass_hat_k": 0.9375,
          "pass_hat_k_ci": [
            0.8625,
            1
          ]
        },
        {
          "k": 5,
          "pass_at_k": 0.9746,
          "pass_at_k_ci": [
            0.9237,
            1
          ],
          "pass_hat_k": 0.9344,
          "pass_hat_k_ci": [
            0.8531,
            1
          ]
        },
        {
          "k": 6,
          "pass_at_k": 0.975,
          "pass_at_k_ci": [
            0.925,
            1
          ],
          "pass_hat_k": 0.9313,
          "pass_hat_k_ci": [
            0.85,
            1
          ]
        },
        {
          "k": 7,
          "pass_at_k": 0.975,
          "pass_at_k_ci": [
            0.925,
            1
          ],
          "pass_hat_k": 0.9281,
          "pass_hat_k_ci": [
            0.85,
            1
          ]
        },
        {
          "k": 8,
          "pass_at_k": 0.975,
          "pass_at_k_ci": [
            0.925,
            1
          ],
          "pass_hat_k": 0.925,
          "pass_hat_k_ci": [
            0.85,
            1
          ]
        }
      ],
      "by_category": {
        "reasoning": {
          "rate": 0.8375,
          "ci": [
            0.7416,
            0.9025
          ],
          "n": 80
        },
        "extraction": {
          "rate": 1,
          "ci": [
            0.9542,
            1
          ],
          "n": 80
        },
        "instruction": {
          "rate": 0.9875,
          "ci": [
            0.9325,
            0.9978
          ],
          "n": 80
        },
        "json": {
          "rate": 1,
          "ci": [
            0.9542,
            1
          ],
          "n": 80
        }
      },
      "flaky": [
        "reasoning-08",
        "instruction-03"
      ],
      "tasks": [
        {
          "id": "reasoning-01",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-02",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-03",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-04",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-05",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-06",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-07",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-08",
          "category": "reasoning",
          "n": 8,
          "c": 3,
          "errors": 0
        },
        {
          "id": "reasoning-09",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-10",
          "category": "reasoning",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "extraction-01",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-02",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-03",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-04",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-05",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-06",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-07",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-08",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-09",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-10",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-01",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-02",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-03",
          "category": "instruction",
          "n": 8,
          "c": 7,
          "errors": 0
        },
        {
          "id": "instruction-04",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-05",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-06",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-07",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-08",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-09",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-10",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-01",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-02",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-03",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-04",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-05",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-06",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-07",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-08",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-09",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-10",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        }
      ],
      "calls": 320,
      "errors": 0,
      "latency_ms": {
        "p50": 891,
        "p95": 1588,
        "mean": 1009
      },
      "tokens": {
        "input": 31720,
        "output": 14303
      },
      "cost_usd": 0.20647,
      "cost_coverage": 1
    },
    {
      "model": "anthropic/claude-haiku-5.5",
      "pass_at_1": {
        "rate": 0.9344,
        "ci": [
          0.9018,
          0.9567
        ],
        "n": 320
      },
      "curve": [
        {
          "k": 1,
          "pass_at_k": 0.9344,
          "pass_at_k_ci": [
            0.8594,
            1
          ],
          "pass_hat_k": 0.9344,
          "pass_hat_k_ci": [
            0.85,
            1
          ]
        },
        {
          "k": 2,
          "pass_at_k": 0.9411,
          "pass_at_k_ci": [
            0.8661,
            1
          ],
          "pass_hat_k": 0.9277,
          "pass_hat_k_ci": [
            0.8384,
            1
          ]
        },
        {
          "k": 3,
          "pass_at_k": 0.9455,
          "pass_at_k_ci": [
            0.8705,
            1
          ],
          "pass_hat_k": 0.9254,
          "pass_hat_k_ci": [
            0.85,
            1
          ]
        },
        {
          "k": 4,
          "pass_at_k": 0.9482,
          "pass_at_k_ci": [
            0.8732,
            1
          ],
          "pass_hat_k": 0.925,
          "pass_hat_k_ci": [
            0.85,
            1
          ]
        },
        {
          "k": 5,
          "pass_at_k": 0.9496,
          "pass_at_k_ci": [
            0.8741,
            1
          ],
          "pass_hat_k": 0.925,
          "pass_hat_k_ci": [
            0.85,
            1
          ]
        },
        {
          "k": 6,
          "pass_at_k": 0.95,
          "pass_at_k_ci": [
            0.875,
            1
          ],
          "pass_hat_k": 0.925,
          "pass_hat_k_ci": [
            0.825,
            1
          ]
        },
        {
          "k": 7,
          "pass_at_k": 0.95,
          "pass_at_k_ci": [
            0.875,
            1
          ],
          "pass_hat_k": 0.925,
          "pass_hat_k_ci": [
            0.825,
            1
          ]
        },
        {
          "k": 8,
          "pass_at_k": 0.95,
          "pass_at_k_ci": [
            0.875,
            1
          ],
          "pass_hat_k": 0.925,
          "pass_hat_k_ci": [
            0.85,
            1
          ]
        }
      ],
      "by_category": {
        "reasoning": {
          "rate": 1,
          "ci": [
            0.9542,
            1
          ],
          "n": 80
        },
        "extraction": {
          "rate": 1,
          "ci": [
            0.9542,
            1
          ],
          "n": 80
        },
        "instruction": {
          "rate": 0.7375,
          "ci": [
            0.6318,
            0.8214
          ],
          "n": 80
        },
        "json": {
          "rate": 1,
          "ci": [
            0.9542,
            1
          ],
          "n": 80
        }
      },
      "flaky": [
        "instruction-08"
      ],
      "tasks": [
        {
          "id": "reasoning-01",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-02",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-03",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-04",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-05",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-06",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-07",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-08",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-09",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-10",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-01",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-02",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-03",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-04",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-05",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-06",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-07",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-08",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-09",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-10",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-01",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-02",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-03",
          "category": "instruction",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "instruction-04",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-05",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-06",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-07",
          "category": "instruction",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "instruction-08",
          "category": "instruction",
          "n": 8,
          "c": 3,
          "errors": 0
        },
        {
          "id": "instruction-09",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-10",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-01",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-02",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-03",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-04",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-05",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-06",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-07",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-08",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-09",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-10",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        }
      ],
      "calls": 320,
      "errors": 0,
      "latency_ms": {
        "p50": 841,
        "p95": 1395,
        "mean": 876
      },
      "tokens": {
        "input": 31720,
        "output": 14206
      },
      "cost_usd": 0.010275,
      "cost_coverage": 1
    },
    {
      "model": "meta/llama-3.3-70b",
      "pass_at_1": {
        "rate": 0.9,
        "ci": [
          0.8622,
          0.9283
        ],
        "n": 320
      },
      "curve": [
        {
          "k": 1,
          "pass_at_k": 0.9,
          "pass_at_k_ci": [
            0.8,
            0.975
          ],
          "pass_hat_k": 0.9,
          "pass_hat_k_ci": [
            0.8,
            0.975
          ]
        },
        {
          "k": 2,
          "pass_at_k": 0.9,
          "pass_at_k_ci": [
            0.8,
            0.975
          ],
          "pass_hat_k": 0.9,
          "pass_hat_k_ci": [
            0.8,
            0.975
          ]
        },
        {
          "k": 3,
          "pass_at_k": 0.9,
          "pass_at_k_ci": [
            0.8,
            0.975
          ],
          "pass_hat_k": 0.9,
          "pass_hat_k_ci": [
            0.8,
            0.975
          ]
        },
        {
          "k": 4,
          "pass_at_k": 0.9,
          "pass_at_k_ci": [
            0.8,
            0.975
          ],
          "pass_hat_k": 0.9,
          "pass_hat_k_ci": [
            0.8,
            0.975
          ]
        },
        {
          "k": 5,
          "pass_at_k": 0.9,
          "pass_at_k_ci": [
            0.8,
            0.975
          ],
          "pass_hat_k": 0.9,
          "pass_hat_k_ci": [
            0.8,
            0.975
          ]
        },
        {
          "k": 6,
          "pass_at_k": 0.9,
          "pass_at_k_ci": [
            0.8,
            0.975
          ],
          "pass_hat_k": 0.9,
          "pass_hat_k_ci": [
            0.8,
            0.975
          ]
        },
        {
          "k": 7,
          "pass_at_k": 0.9,
          "pass_at_k_ci": [
            0.8,
            0.975
          ],
          "pass_hat_k": 0.9,
          "pass_hat_k_ci": [
            0.8,
            0.975
          ]
        },
        {
          "k": 8,
          "pass_at_k": 0.9,
          "pass_at_k_ci": [
            0.8,
            0.975
          ],
          "pass_hat_k": 0.9,
          "pass_hat_k_ci": [
            0.8,
            0.975
          ]
        }
      ],
      "by_category": {
        "reasoning": {
          "rate": 0.7,
          "ci": [
            0.5923,
            0.7894
          ],
          "n": 80
        },
        "extraction": {
          "rate": 1,
          "ci": [
            0.9542,
            1
          ],
          "n": 80
        },
        "instruction": {
          "rate": 0.9,
          "ci": [
            0.8149,
            0.9485
          ],
          "n": 80
        },
        "json": {
          "rate": 1,
          "ci": [
            0.9542,
            1
          ],
          "n": 80
        }
      },
      "flaky": [],
      "tasks": [
        {
          "id": "reasoning-01",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-02",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-03",
          "category": "reasoning",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "reasoning-04",
          "category": "reasoning",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "reasoning-05",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-06",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-07",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-08",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-09",
          "category": "reasoning",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "reasoning-10",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-01",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-02",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-03",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-04",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-05",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-06",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-07",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-08",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-09",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-10",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-01",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-02",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-03",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-04",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-05",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-06",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-07",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-08",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-09",
          "category": "instruction",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "instruction-10",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-01",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-02",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-03",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-04",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-05",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-06",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-07",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-08",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-09",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-10",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        }
      ],
      "calls": 320,
      "errors": 0,
      "latency_ms": {
        "p50": 393,
        "p95": 838,
        "mean": 461
      },
      "tokens": {
        "input": 30616,
        "output": 6108
      },
      "cost_usd": 0.026441,
      "cost_coverage": 1
    },
    {
      "model": "deepseek/deepseek-v3.2",
      "pass_at_1": {
        "rate": 0.9438,
        "ci": [
          0.9128,
          0.9641
        ],
        "n": 320
      },
      "curve": [
        {
          "k": 1,
          "pass_at_k": 0.9438,
          "pass_at_k_ci": [
            0.8844,
            0.9906
          ],
          "pass_hat_k": 0.9438,
          "pass_hat_k_ci": [
            0.8813,
            0.9906
          ]
        },
        {
          "k": 2,
          "pass_at_k": 0.9661,
          "pass_at_k_ci": [
            0.9071,
            0.9991
          ],
          "pass_hat_k": 0.9214,
          "pass_hat_k_ci": [
            0.8455,
            0.9839
          ]
        },
        {
          "k": 3,
          "pass_at_k": 0.9728,
          "pass_at_k_ci": [
            0.921,
            1
          ],
          "pass_hat_k": 0.9058,
          "pass_hat_k_ci": [
            0.8188,
            0.9813
          ]
        },
        {
          "k": 4,
          "pass_at_k": 0.9746,
          "pass_at_k_ci": [
            0.9243,
            1
          ],
          "pass_hat_k": 0.895,
          "pass_hat_k_ci": [
            0.8093,
            0.9768
          ]
        },
        {
          "k": 5,
          "pass_at_k": 0.975,
          "pass_at_k_ci": [
            0.925,
            1
          ],
          "pass_hat_k": 0.8875,
          "pass_hat_k_ci": [
            0.7853,
            0.975
          ]
        },
        {
          "k": 6,
          "pass_at_k": 0.975,
          "pass_at_k_ci": [
            0.925,
            1
          ],
          "pass_hat_k": 0.8821,
          "pass_hat_k_ci": [
            0.7705,
            0.975
          ]
        },
        {
          "k": 7,
          "pass_at_k": 0.975,
          "pass_at_k_ci": [
            0.925,
            1
          ],
          "pass_hat_k": 0.8781,
          "pass_hat_k_ci": [
            0.775,
            0.975
          ]
        },
        {
          "k": 8,
          "pass_at_k": 0.975,
          "pass_at_k_ci": [
            0.925,
            1
          ],
          "pass_hat_k": 0.875,
          "pass_hat_k_ci": [
            0.775,
            0.975
          ]
        }
      ],
      "by_category": {
        "reasoning": {
          "rate": 0.95,
          "ci": [
            0.8784,
            0.9804
          ],
          "n": 80
        },
        "extraction": {
          "rate": 1,
          "ci": [
            0.9542,
            1
          ],
          "n": 80
        },
        "instruction": {
          "rate": 0.8375,
          "ci": [
            0.7416,
            0.9025
          ],
          "n": 80
        },
        "json": {
          "rate": 0.9875,
          "ci": [
            0.9325,
            0.9978
          ],
          "n": 80
        }
      },
      "flaky": [
        "reasoning-08",
        "instruction-03",
        "instruction-09",
        "json-09"
      ],
      "tasks": [
        {
          "id": "reasoning-01",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-02",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-03",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-04",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-05",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-06",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-07",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-08",
          "category": "reasoning",
          "n": 8,
          "c": 4,
          "errors": 0
        },
        {
          "id": "reasoning-09",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-10",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-01",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-02",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-03",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-04",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-05",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-06",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-07",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-08",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-09",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-10",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-01",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-02",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-03",
          "category": "instruction",
          "n": 8,
          "c": 5,
          "errors": 0
        },
        {
          "id": "instruction-04",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-05",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-06",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-07",
          "category": "instruction",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "instruction-08",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-09",
          "category": "instruction",
          "n": 8,
          "c": 6,
          "errors": 0
        },
        {
          "id": "instruction-10",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-01",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-02",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-03",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-04",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-05",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-06",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-07",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-08",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-09",
          "category": "json",
          "n": 8,
          "c": 7,
          "errors": 0
        },
        {
          "id": "json-10",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        }
      ],
      "calls": 320,
      "errors": 0,
      "latency_ms": {
        "p50": 855,
        "p95": 3114,
        "mean": 1230
      },
      "tokens": {
        "input": 20856,
        "output": 5805
      },
      "cost_usd": 0.02367,
      "cost_coverage": 1
    },
    {
      "model": "openai/gpt-4.1-mini",
      "pass_at_1": {
        "rate": 0.8813,
        "ci": [
          0.8412,
          0.9122
        ],
        "n": 320
      },
      "curve": [
        {
          "k": 1,
          "pass_at_k": 0.8813,
          "pass_at_k_ci": [
            0.775,
            0.975
          ],
          "pass_hat_k": 0.8813,
          "pass_hat_k_ci": [
            0.7813,
            0.975
          ]
        },
        {
          "k": 2,
          "pass_at_k": 0.8866,
          "pass_at_k_ci": [
            0.7866,
            0.975
          ],
          "pass_hat_k": 0.8759,
          "pass_hat_k_ci": [
            0.7536,
            0.975
          ]
        },
        {
          "k": 3,
          "pass_at_k": 0.8911,
          "pass_at_k_ci": [
            0.7911,
            0.975
          ],
          "pass_hat_k": 0.875,
          "pass_hat_k_ci": [
            0.775,
            0.975
          ]
        },
        {
          "k": 4,
          "pass_at_k": 0.8946,
          "pass_at_k_ci": [
            0.7946,
            0.975
          ],
          "pass_hat_k": 0.875,
          "pass_hat_k_ci": [
            0.775,
            0.975
          ]
        },
        {
          "k": 5,
          "pass_at_k": 0.8973,
          "pass_at_k_ci": [
            0.7973,
            0.975
          ],
          "pass_hat_k": 0.875,
          "pass_hat_k_ci": [
            0.775,
            0.975
          ]
        },
        {
          "k": 6,
          "pass_at_k": 0.8991,
          "pass_at_k_ci": [
            0.7991,
            0.975
          ],
          "pass_hat_k": 0.875,
          "pass_hat_k_ci": [
            0.775,
            0.975
          ]
        },
        {
          "k": 7,
          "pass_at_k": 0.9,
          "pass_at_k_ci": [
            0.8,
            0.975
          ],
          "pass_hat_k": 0.875,
          "pass_hat_k_ci": [
            0.775,
            0.975
          ]
        },
        {
          "k": 8,
          "pass_at_k": 0.9,
          "pass_at_k_ci": [
            0.8,
            0.975
          ],
          "pass_hat_k": 0.875,
          "pass_hat_k_ci": [
            0.75,
            0.975
          ]
        }
      ],
      "by_category": {
        "reasoning": {
          "rate": 0.8,
          "ci": [
            0.6995,
            0.873
          ],
          "n": 80
        },
        "extraction": {
          "rate": 1,
          "ci": [
            0.9542,
            1
          ],
          "n": 80
        },
        "instruction": {
          "rate": 0.725,
          "ci": [
            0.6186,
            0.8108
          ],
          "n": 80
        },
        "json": {
          "rate": 1,
          "ci": [
            0.9542,
            1
          ],
          "n": 80
        }
      },
      "flaky": [
        "instruction-09"
      ],
      "tasks": [
        {
          "id": "reasoning-01",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-02",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-03",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-04",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-05",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-06",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-07",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-08",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-09",
          "category": "reasoning",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "reasoning-10",
          "category": "reasoning",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "extraction-01",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-02",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-03",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-04",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-05",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-06",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-07",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-08",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-09",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-10",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-01",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-02",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-03",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-04",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-05",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-06",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-07",
          "category": "instruction",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "instruction-08",
          "category": "instruction",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "instruction-09",
          "category": "instruction",
          "n": 8,
          "c": 2,
          "errors": 0
        },
        {
          "id": "instruction-10",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-01",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-02",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-03",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-04",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-05",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-06",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-07",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-08",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-09",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-10",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        }
      ],
      "calls": 320,
      "errors": 0,
      "latency_ms": {
        "p50": 736,
        "p95": 1243,
        "mean": 794
      },
      "tokens": {
        "input": 22552,
        "output": 5237
      },
      "cost_usd": 0.0174,
      "cost_coverage": 1
    },
    {
      "model": "google/gemini-2.5-flash",
      "pass_at_1": {
        "rate": 0.9469,
        "ci": [
          0.9166,
          0.9666
        ],
        "n": 320
      },
      "curve": [
        {
          "k": 1,
          "pass_at_k": 0.9469,
          "pass_at_k_ci": [
            0.8969,
            0.9844
          ],
          "pass_hat_k": 0.9469,
          "pass_hat_k_ci": [
            0.8969,
            0.9844
          ]
        },
        {
          "k": 2,
          "pass_at_k": 0.9804,
          "pass_at_k_ci": [
            0.9482,
            1
          ],
          "pass_hat_k": 0.9134,
          "pass_hat_k_ci": [
            0.842,
            0.9696
          ]
        },
        {
          "k": 3,
          "pass_at_k": 0.9893,
          "pass_at_k_ci": [
            0.9696,
            1
          ],
          "pass_hat_k": 0.8888,
          "pass_hat_k_ci": [
            0.8013,
            0.958
          ]
        },
        {
          "k": 4,
          "pass_at_k": 0.9943,
          "pass_at_k_ci": [
            0.9832,
            1
          ],
          "pass_hat_k": 0.8682,
          "pass_hat_k_ci": [
            0.7689,
            0.95
          ]
        },
        {
          "k": 5,
          "pass_at_k": 0.9973,
          "pass_at_k_ci": [
            0.992,
            1
          ],
          "pass_hat_k": 0.8496,
          "pass_hat_k_ci": [
            0.7473,
            0.9375
          ]
        },
        {
          "k": 6,
          "pass_at_k": 0.9991,
          "pass_at_k_ci": [
            0.9973,
            1
          ],
          "pass_hat_k": 0.8321,
          "pass_hat_k_ci": [
            0.7277,
            0.925
          ]
        },
        {
          "k": 7,
          "pass_at_k": 1,
          "pass_at_k_ci": [
            1,
            1
          ],
          "pass_hat_k": 0.8156,
          "pass_hat_k_ci": [
            0.7,
            0.9125
          ]
        },
        {
          "k": 8,
          "pass_at_k": 1,
          "pass_at_k_ci": [
            1,
            1
          ],
          "pass_hat_k": 0.8,
          "pass_hat_k_ci": [
            0.675,
            0.925
          ]
        }
      ],
      "by_category": {
        "reasoning": {
          "rate": 0.9875,
          "ci": [
            0.9325,
            0.9978
          ],
          "n": 80
        },
        "extraction": {
          "rate": 0.9875,
          "ci": [
            0.9325,
            0.9978
          ],
          "n": 80
        },
        "instruction": {
          "rate": 0.8375,
          "ci": [
            0.7416,
            0.9025
          ],
          "n": 80
        },
        "json": {
          "rate": 0.975,
          "ci": [
            0.9134,
            0.9931
          ],
          "n": 80
        }
      },
      "flaky": [
        "reasoning-03",
        "extraction-10",
        "instruction-01",
        "instruction-03",
        "instruction-07",
        "instruction-08",
        "json-08",
        "json-10"
      ],
      "tasks": [
        {
          "id": "reasoning-01",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-02",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-03",
          "category": "reasoning",
          "n": 8,
          "c": 7,
          "errors": 0
        },
        {
          "id": "reasoning-04",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-05",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-06",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-07",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-08",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-09",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-10",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-01",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-02",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-03",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-04",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-05",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-06",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-07",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-08",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-09",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-10",
          "category": "extraction",
          "n": 8,
          "c": 7,
          "errors": 0
        },
        {
          "id": "instruction-01",
          "category": "instruction",
          "n": 8,
          "c": 7,
          "errors": 0
        },
        {
          "id": "instruction-02",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-03",
          "category": "instruction",
          "n": 8,
          "c": 6,
          "errors": 0
        },
        {
          "id": "instruction-04",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-05",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-06",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-07",
          "category": "instruction",
          "n": 8,
          "c": 2,
          "errors": 0
        },
        {
          "id": "instruction-08",
          "category": "instruction",
          "n": 8,
          "c": 4,
          "errors": 0
        },
        {
          "id": "instruction-09",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-10",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-01",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-02",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-03",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-04",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-05",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-06",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-07",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-08",
          "category": "json",
          "n": 8,
          "c": 7,
          "errors": 0
        },
        {
          "id": "json-09",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-10",
          "category": "json",
          "n": 8,
          "c": 7,
          "errors": 0
        }
      ],
      "calls": 320,
      "errors": 0,
      "latency_ms": {
        "p50": 1440,
        "p95": 2423,
        "mean": 1530
      },
      "tokens": {
        "input": 20576,
        "output": 56125
      },
      "cost_usd": 0.146485,
      "cost_coverage": 1
    },
    {
      "model": "mistral/mistral-small",
      "pass_at_1": {
        "rate": 0.8625,
        "ci": [
          0.8204,
          0.896
        ],
        "n": 320
      },
      "curve": [
        {
          "k": 1,
          "pass_at_k": 0.8625,
          "pass_at_k_ci": [
            0.7531,
            0.9625
          ],
          "pass_hat_k": 0.8625,
          "pass_hat_k_ci": [
            0.75,
            0.9594
          ]
        },
        {
          "k": 2,
          "pass_at_k": 0.8741,
          "pass_at_k_ci": [
            0.75,
            0.9732
          ],
          "pass_hat_k": 0.8509,
          "pass_hat_k_ci": [
            0.7455,
            0.9438
          ]
        },
        {
          "k": 3,
          "pass_at_k": 0.875,
          "pass_at_k_ci": [
            0.775,
            0.975
          ],
          "pass_hat_k": 0.8402,
          "pass_hat_k_ci": [
            0.7308,
            0.9406
          ]
        },
        {
          "k": 4,
          "pass_at_k": 0.875,
          "pass_at_k_ci": [
            0.75,
            0.975
          ],
          "pass_hat_k": 0.8304,
          "pass_hat_k_ci": [
            0.7179,
            0.9304
          ]
        },
        {
          "k": 5,
          "pass_at_k": 0.875,
          "pass_at_k_ci": [
            0.75,
            0.975
          ],
          "pass_hat_k": 0.8214,
          "pass_hat_k_ci": [
            0.6991,
            0.9277
          ]
        },
        {
          "k": 6,
          "pass_at_k": 0.875,
          "pass_at_k_ci": [
            0.75,
            0.975
          ],
          "pass_hat_k": 0.8134,
          "pass_hat_k_ci": [
            0.6893,
            0.9188
          ]
        },
        {
          "k": 7,
          "pass_at_k": 0.875,
          "pass_at_k_ci": [
            0.75,
            0.975
          ],
          "pass_hat_k": 0.8063,
          "pass_hat_k_ci": [
            0.6813,
            0.925
          ]
        },
        {
          "k": 8,
          "pass_at_k": 0.875,
          "pass_at_k_ci": [
            0.75,
            0.975
          ],
          "pass_hat_k": 0.8,
          "pass_hat_k_ci": [
            0.675,
            0.925
          ]
        }
      ],
      "by_category": {
        "reasoning": {
          "rate": 0.7625,
          "ci": [
            0.6586,
            0.8424
          ],
          "n": 80
        },
        "extraction": {
          "rate": 1,
          "ci": [
            0.9542,
            1
          ],
          "n": 80
        },
        "instruction": {
          "rate": 0.6875,
          "ci": [
            0.5793,
            0.7785
          ],
          "n": 80
        },
        "json": {
          "rate": 1,
          "ci": [
            0.9542,
            1
          ],
          "n": 80
        }
      },
      "flaky": [
        "reasoning-03",
        "reasoning-04",
        "instruction-03"
      ],
      "tasks": [
        {
          "id": "reasoning-01",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-02",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-03",
          "category": "reasoning",
          "n": 8,
          "c": 6,
          "errors": 0
        },
        {
          "id": "reasoning-04",
          "category": "reasoning",
          "n": 8,
          "c": 7,
          "errors": 0
        },
        {
          "id": "reasoning-05",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-06",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-07",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-08",
          "category": "reasoning",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "reasoning-09",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-10",
          "category": "reasoning",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "extraction-01",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-02",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-03",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-04",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-05",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-06",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-07",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-08",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-09",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-10",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-01",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-02",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-03",
          "category": "instruction",
          "n": 8,
          "c": 7,
          "errors": 0
        },
        {
          "id": "instruction-04",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-05",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-06",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-07",
          "category": "instruction",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "instruction-08",
          "category": "instruction",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "instruction-09",
          "category": "instruction",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "instruction-10",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-01",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-02",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-03",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-04",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-05",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-06",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-07",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-08",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-09",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-10",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        }
      ],
      "calls": 320,
      "errors": 0,
      "latency_ms": {
        "p50": 600,
        "p95": 1286,
        "mean": 682
      },
      "tokens": {
        "input": 25576,
        "output": 6273
      },
      "cost_usd": 0.005462,
      "cost_coverage": 1
    },
    {
      "model": "alibaba/qwen-3-32b",
      "pass_at_1": {
        "rate": 0.7906,
        "ci": [
          0.7427,
          0.8316
        ],
        "n": 320
      },
      "curve": [
        {
          "k": 1,
          "pass_at_k": 0.7906,
          "pass_at_k_ci": [
            0.6781,
            0.8938
          ],
          "pass_hat_k": 0.7906,
          "pass_hat_k_ci": [
            0.6781,
            0.8938
          ]
        },
        {
          "k": 2,
          "pass_at_k": 0.8321,
          "pass_at_k_ci": [
            0.7241,
            0.9259
          ],
          "pass_hat_k": 0.7491,
          "pass_hat_k_ci": [
            0.6161,
            0.8625
          ]
        },
        {
          "k": 3,
          "pass_at_k": 0.8531,
          "pass_at_k_ci": [
            0.7429,
            0.9442
          ],
          "pass_hat_k": 0.7286,
          "pass_hat_k_ci": [
            0.5987,
            0.8496
          ]
        },
        {
          "k": 4,
          "pass_at_k": 0.8675,
          "pass_at_k_ci": [
            0.7618,
            0.9532
          ],
          "pass_hat_k": 0.7146,
          "pass_hat_k_ci": [
            0.5757,
            0.8396
          ]
        },
        {
          "k": 5,
          "pass_at_k": 0.8781,
          "pass_at_k_ci": [
            0.7777,
            0.9625
          ],
          "pass_hat_k": 0.7036,
          "pass_hat_k_ci": [
            0.5629,
            0.8344
          ]
        },
        {
          "k": 6,
          "pass_at_k": 0.8866,
          "pass_at_k_ci": [
            0.7866,
            0.9679
          ],
          "pass_hat_k": 0.6938,
          "pass_hat_k_ci": [
            0.55,
            0.825
          ]
        },
        {
          "k": 7,
          "pass_at_k": 0.8938,
          "pass_at_k_ci": [
            0.7906,
            0.9719
          ],
          "pass_hat_k": 0.6844,
          "pass_hat_k_ci": [
            0.5406,
            0.8281
          ]
        },
        {
          "k": 8,
          "pass_at_k": 0.9,
          "pass_at_k_ci": [
            0.8,
            0.975
          ],
          "pass_hat_k": 0.675,
          "pass_hat_k_ci": [
            0.525,
            0.825
          ]
        }
      ],
      "by_category": {
        "reasoning": {
          "rate": 0.5125,
          "ci": [
            0.4049,
            0.6189
          ],
          "n": 80
        },
        "extraction": {
          "rate": 1,
          "ci": [
            0.9542,
            1
          ],
          "n": 80
        },
        "instruction": {
          "rate": 0.6875,
          "ci": [
            0.5793,
            0.7785
          ],
          "n": 80
        },
        "json": {
          "rate": 0.9625,
          "ci": [
            0.8955,
            0.9872
          ],
          "n": 80
        }
      },
      "flaky": [
        "reasoning-01",
        "reasoning-02",
        "reasoning-05",
        "reasoning-09",
        "reasoning-10",
        "instruction-03",
        "instruction-06",
        "instruction-08",
        "json-09"
      ],
      "tasks": [
        {
          "id": "reasoning-01",
          "category": "reasoning",
          "n": 8,
          "c": 4,
          "errors": 0
        },
        {
          "id": "reasoning-02",
          "category": "reasoning",
          "n": 8,
          "c": 7,
          "errors": 0
        },
        {
          "id": "reasoning-03",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-04",
          "category": "reasoning",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "reasoning-05",
          "category": "reasoning",
          "n": 8,
          "c": 1,
          "errors": 0
        },
        {
          "id": "reasoning-06",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-07",
          "category": "reasoning",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "reasoning-08",
          "category": "reasoning",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "reasoning-09",
          "category": "reasoning",
          "n": 8,
          "c": 2,
          "errors": 0
        },
        {
          "id": "reasoning-10",
          "category": "reasoning",
          "n": 8,
          "c": 3,
          "errors": 0
        },
        {
          "id": "extraction-01",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-02",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-03",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-04",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-05",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-06",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-07",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-08",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-09",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-10",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-01",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-02",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-03",
          "category": "instruction",
          "n": 8,
          "c": 7,
          "errors": 0
        },
        {
          "id": "instruction-04",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-05",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-06",
          "category": "instruction",
          "n": 8,
          "c": 7,
          "errors": 0
        },
        {
          "id": "instruction-07",
          "category": "instruction",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "instruction-08",
          "category": "instruction",
          "n": 8,
          "c": 1,
          "errors": 0
        },
        {
          "id": "instruction-09",
          "category": "instruction",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "instruction-10",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-01",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-02",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-03",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-04",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-05",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-06",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-07",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-08",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-09",
          "category": "json",
          "n": 8,
          "c": 5,
          "errors": 0
        },
        {
          "id": "json-10",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        }
      ],
      "calls": 320,
      "errors": 0,
      "latency_ms": {
        "p50": 397,
        "p95": 720,
        "mean": 434
      },
      "tokens": {
        "input": 25440,
        "output": 6901
      },
      "cost_usd": 0.007957,
      "cost_coverage": 1
    },
    {
      "model": "meta/llama-3.1-8b",
      "pass_at_1": {
        "rate": 0.6938,
        "ci": [
          0.6412,
          0.7417
        ],
        "n": 320
      },
      "curve": [
        {
          "k": 1,
          "pass_at_k": 0.6938,
          "pass_at_k_ci": [
            0.5563,
            0.8219
          ],
          "pass_hat_k": 0.6938,
          "pass_hat_k_ci": [
            0.5594,
            0.8219
          ]
        },
        {
          "k": 2,
          "pass_at_k": 0.7259,
          "pass_at_k_ci": [
            0.5839,
            0.8536
          ],
          "pass_hat_k": 0.6616,
          "pass_hat_k_ci": [
            0.5241,
            0.7973
          ]
        },
        {
          "k": 3,
          "pass_at_k": 0.7411,
          "pass_at_k_ci": [
            0.6161,
            0.8656
          ],
          "pass_hat_k": 0.6446,
          "pass_hat_k_ci": [
            0.5031,
            0.7897
          ]
        },
        {
          "k": 4,
          "pass_at_k": 0.7518,
          "pass_at_k_ci": [
            0.6196,
            0.8714
          ],
          "pass_hat_k": 0.6321,
          "pass_hat_k_ci": [
            0.4875,
            0.7679
          ]
        },
        {
          "k": 5,
          "pass_at_k": 0.7603,
          "pass_at_k_ci": [
            0.6232,
            0.8786
          ],
          "pass_hat_k": 0.6219,
          "pass_hat_k_ci": [
            0.4688,
            0.7625
          ]
        },
        {
          "k": 6,
          "pass_at_k": 0.767,
          "pass_at_k_ci": [
            0.6295,
            0.8866
          ],
          "pass_hat_k": 0.6134,
          "pass_hat_k_ci": [
            0.4643,
            0.7571
          ]
        },
        {
          "k": 7,
          "pass_at_k": 0.7719,
          "pass_at_k_ci": [
            0.625,
            0.8969
          ],
          "pass_hat_k": 0.6063,
          "pass_hat_k_ci": [
            0.4531,
            0.7531
          ]
        },
        {
          "k": 8,
          "pass_at_k": 0.775,
          "pass_at_k_ci": [
            0.65,
            0.9
          ],
          "pass_hat_k": 0.6,
          "pass_hat_k_ci": [
            0.45,
            0.75
          ]
        }
      ],
      "by_category": {
        "reasoning": {
          "rate": 0.1375,
          "ci": [
            0.0785,
            0.2297
          ],
          "n": 80
        },
        "extraction": {
          "rate": 1,
          "ci": [
            0.9542,
            1
          ],
          "n": 80
        },
        "instruction": {
          "rate": 0.8375,
          "ci": [
            0.7416,
            0.9025
          ],
          "n": 80
        },
        "json": {
          "rate": 0.8,
          "ci": [
            0.6995,
            0.873
          ],
          "n": 80
        }
      },
      "flaky": [
        "reasoning-02",
        "reasoning-06",
        "reasoning-07",
        "reasoning-08",
        "instruction-03",
        "instruction-06",
        "instruction-07"
      ],
      "tasks": [
        {
          "id": "reasoning-01",
          "category": "reasoning",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "reasoning-02",
          "category": "reasoning",
          "n": 8,
          "c": 1,
          "errors": 0
        },
        {
          "id": "reasoning-03",
          "category": "reasoning",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "reasoning-04",
          "category": "reasoning",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "reasoning-05",
          "category": "reasoning",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "reasoning-06",
          "category": "reasoning",
          "n": 8,
          "c": 6,
          "errors": 0
        },
        {
          "id": "reasoning-07",
          "category": "reasoning",
          "n": 8,
          "c": 2,
          "errors": 0
        },
        {
          "id": "reasoning-08",
          "category": "reasoning",
          "n": 8,
          "c": 2,
          "errors": 0
        },
        {
          "id": "reasoning-09",
          "category": "reasoning",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "reasoning-10",
          "category": "reasoning",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "extraction-01",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-02",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-03",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-04",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-05",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-06",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-07",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-08",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-09",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "extraction-10",
          "category": "extraction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-01",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-02",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-03",
          "category": "instruction",
          "n": 8,
          "c": 7,
          "errors": 0
        },
        {
          "id": "instruction-04",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-05",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-06",
          "category": "instruction",
          "n": 8,
          "c": 7,
          "errors": 0
        },
        {
          "id": "instruction-07",
          "category": "instruction",
          "n": 8,
          "c": 5,
          "errors": 0
        },
        {
          "id": "instruction-08",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "instruction-09",
          "category": "instruction",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "instruction-10",
          "category": "instruction",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-01",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-02",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-03",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-04",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-05",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-06",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-07",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        },
        {
          "id": "json-08",
          "category": "json",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "json-09",
          "category": "json",
          "n": 8,
          "c": 0,
          "errors": 0
        },
        {
          "id": "json-10",
          "category": "json",
          "n": 8,
          "c": 8,
          "errors": 0
        }
      ],
      "calls": 320,
      "errors": 0,
      "latency_ms": {
        "p50": 372,
        "p95": 543,
        "mean": 392
      },
      "tokens": {
        "input": 24216,
        "output": 6315
      },
      "cost_usd": 0.006717,
      "cost_coverage": 1
    }
  ]
}