{
 "name": "PWS-Decisions",
 "version": "0.1.0-rc1",
 "questions_sha256": "721900f4bf28f86981b5af78b0bcadff4c424eb5261e9e9b97c16030f7ae3fb5",
 "questions": 100,
 "language": "English",
 "random_accuracy": 0.44666666666666666,
 "choice_counts": {
  "2": 77,
  "3": 5,
  "4": 18
 },
 "bootstrap": {
  "resamples": 10000,
  "seed": 20261105,
  "unit": "question",
  "percentiles": [
   2.5,
   97.5
  ],
  "score": "a question's mean correctness over the setup's runs"
 },
 "harness": "https://github.com/parallel-web/pws-decisions-benchmark",
 "runs_per_setup": 3,
 "one_question_per_request": true,
 "timeout_seconds": 60.0,
 "retries": {
  "statuses": "408, 425, 429 and 5xx, and transport failures",
  "max": 5
 },
 "percentile": 0.95,
 "sure_threshold": 0.99,
 "dates": [
  "2026-11-05",
  "2026-11-06"
 ],
 "run_cost_usd": 258.433588631,
 "setups": [
  {
   "provider": "perplexity",
   "model": "pplx-decider-v1.2",
   "perplexity_mode": null,
   "request_model": "pplx-decider-v1.2",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "Perplexity's Decisions API price",
    "source": "https://docs.perplexity.ai/docs/getting-started/pricing",
    "date": "2026-11-05",
    "input_per_million": 0.02,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T15:33:34+00:00",
     "ended": "2026-11-05T15:34:10+00:00",
     "questions": 100,
     "correct": 50,
     "accuracy": 0.5,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.24853690201416612,
     "p95_request_seconds": 0.8767989940242842,
     "input_tokens": 19838,
     "output_tokens": 100,
     "cost_usd": 0.00039676,
     "cost_per_million_questions": 3.9676,
     "served_models": [
      "pplx-decider-v1.2"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6454122760267808,
      "sure": 2,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:34:10+00:00",
     "ended": "2026-11-05T15:34:38+00:00",
     "questions": 100,
     "correct": 51,
     "accuracy": 0.51,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.2401812725001946,
     "p95_request_seconds": 0.3958347919397056,
     "input_tokens": 19838,
     "output_tokens": 100,
     "cost_usd": 0.00039676,
     "cost_per_million_questions": 3.9676,
     "served_models": [
      "pplx-decider-v1.2"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.64618813740891,
      "sure": 2,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:34:38+00:00",
     "ended": "2026-11-05T15:35:12+00:00",
     "questions": 100,
     "correct": 51,
     "accuracy": 0.51,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.24254921055398881,
     "p95_request_seconds": 0.6961708300514147,
     "input_tokens": 19838,
     "output_tokens": 100,
     "cost_usd": 0.00039676,
     "cost_per_million_questions": 3.9676,
     "served_models": [
      "pplx-decider-v1.2"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6464507678213551,
      "sure": 2,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.5066666666666667,
    "min": 0.5,
    "max": 0.51
   },
   "accuracy_ci95": {
    "low": 0.41,
    "high": 0.6
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.2437557950227832,
    "min": 0.2401812725001946,
    "max": 0.24853690201416612
   },
   "p95_request_seconds": {
    "mean": 0.6562682053384682,
    "min": 0.3958347919397056,
    "max": 0.8767989940242842
   },
   "cost_per_million_questions": {
    "mean": 3.9676,
    "min": 3.9676,
    "max": 3.9676
   },
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.6460170604190153,
    "min": 0.6454122760267808,
    "max": 0.6464507678213551
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 2,
    "min": 2,
    "max": 2
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.4766666666666667,
     0.44,
     0.05333333333333334,
     0.03
    ],
    "answers": 300,
    "yes_share": 0.5666666666666667,
    "first_share_other": 0.3416666666666667,
    "baseline": 0.4821873638344227,
    "above": 0.024479302832244032,
    "p": 0.3040488658716918,
    "first_in_10": 5,
    "confidence": {
     "answers": 300,
     "mean_right": 0.7001842540919322,
     "mean_wrong": 0.5903858885387222,
     "sure": 6,
     "sure_wrong": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     0,
     1,
     1
    ],
    "hard_right_all_runs": 0,
    "reversed": {
     "not_run": "no API credit"
    }
   }
  },
  {
   "provider": "perplexity",
   "model": "pplx-decider-v1.2",
   "perplexity_mode": "web_grounding",
   "request_model": "pplx-decider-v1.2",
   "group": "decision",
   "web": true,
   "price": {
    "basis": "Perplexity's Decisions API price with web grounding",
    "source": "https://docs.perplexity.ai/docs/getting-started/pricing",
    "date": "2026-11-05",
    "input_per_million": 0.02,
    "per_request": 0.001
   },
   "runs": [
    {
     "started": "2026-11-05T15:35:12+00:00",
     "ended": "2026-11-05T15:36:05+00:00",
     "questions": 100,
     "correct": 87,
     "accuracy": 0.87,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.4825088730431162,
     "p95_request_seconds": 0.8764734839787707,
     "input_tokens": 204444,
     "output_tokens": 100,
     "cost_usd": 0.10408888000000001,
     "cost_per_million_questions": 1040.8888000000002,
     "served_models": [
      "pplx-decider-v1.2"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9063206230548513,
      "sure": 56,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:36:06+00:00",
     "ended": "2026-11-05T15:36:56+00:00",
     "questions": 100,
     "correct": 87,
     "accuracy": 0.87,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.47519418550655246,
     "p95_request_seconds": 0.6193466630065814,
     "input_tokens": 204865,
     "output_tokens": 100,
     "cost_usd": 0.1040973,
     "cost_per_million_questions": 1040.973,
     "served_models": [
      "pplx-decider-v1.2"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.908087235921923,
      "sure": 56,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:36:56+00:00",
     "ended": "2026-11-05T15:37:48+00:00",
     "questions": 100,
     "correct": 87,
     "accuracy": 0.87,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.49104268953669816,
     "p95_request_seconds": 0.689326073974371,
     "input_tokens": 205483,
     "output_tokens": 100,
     "cost_usd": 0.10410966,
     "cost_per_million_questions": 1041.0966,
     "served_models": [
      "pplx-decider-v1.2"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9107182915758127,
      "sure": 57,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.87,
    "min": 0.87,
    "max": 0.87
   },
   "accuracy_ci95": {
    "low": 0.8,
    "high": 0.93
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.48291524936212227,
    "min": 0.47519418550655246,
    "max": 0.49104268953669816
   },
   "p95_request_seconds": {
    "mean": 0.728382073653241,
    "min": 0.6193466630065814,
    "max": 0.8764734839787707
   },
   "cost_per_million_questions": {
    "mean": 1040.9861333333333,
    "min": 1040.8888000000002,
    "max": 1041.0966
   },
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.908375383517529,
    "min": 0.9063206230548513,
    "max": 0.9107182915758127
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 56.333333333333336,
    "min": 56,
    "max": 57
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.68,
     0.25,
     0.05,
     0.02
    ],
    "answers": 300,
    "yes_share": 0.7666666666666667,
    "first_share_other": 0.55,
    "baseline": 0.5864183006535948,
    "above": 0.2835816993464052,
    "p": 2.18085549619218e-11,
    "first_in_10": 7,
    "confidence": {
     "answers": 300,
     "mean_right": 0.934312858777367,
     "mean_wrong": 0.7347938183170746,
     "sure": 169,
     "sure_wrong": 0
    },
    "explained": false,
    "hard_right": [
     8,
     8,
     8
    ],
    "hard_right_all_runs": 8,
    "reversed": {
     "not_run": "no API credit"
    }
   }
  },
  {
   "provider": "openai-decisions",
   "model": "gpt-6-luna",
   "perplexity_mode": null,
   "request_model": "gpt-6-luna",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "OpenAI's Decisions API price",
    "source": "https://developers.openai.com/api/docs/guides/decisions#pricing-and-availability",
    "date": "2026-10-08",
    "input_per_million": 0.1,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T15:33:34+00:00",
     "ended": "2026-11-05T15:34:05+00:00",
     "questions": 100,
     "correct": 42,
     "accuracy": 0.42,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.2783424025401473,
     "p95_request_seconds": 0.48984210495837033,
     "input_tokens": 21925,
     "output_tokens": 0,
     "cost_usd": 0.0021925,
     "cost_per_million_questions": 21.925,
     "served_models": [
      "gpt-6-luna"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6879,
      "sure": 6,
      "sure_wrong": 3
     }
    },
    {
     "started": "2026-11-05T15:34:05+00:00",
     "ended": "2026-11-05T15:34:36+00:00",
     "questions": 100,
     "correct": 42,
     "accuracy": 0.42,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.2940311215352267,
     "p95_request_seconds": 0.5403023459948599,
     "input_tokens": 21925,
     "output_tokens": 0,
     "cost_usd": 0.0021925,
     "cost_per_million_questions": 21.925,
     "served_models": [
      "gpt-6-luna"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6879,
      "sure": 6,
      "sure_wrong": 3
     }
    },
    {
     "started": "2026-11-05T15:34:37+00:00",
     "ended": "2026-11-05T15:35:07+00:00",
     "questions": 100,
     "correct": 42,
     "accuracy": 0.42,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.28302984905894846,
     "p95_request_seconds": 0.5350455519510433,
     "input_tokens": 21925,
     "output_tokens": 0,
     "cost_usd": 0.0021925,
     "cost_per_million_questions": 21.925,
     "served_models": [
      "gpt-6-luna"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6879,
      "sure": 6,
      "sure_wrong": 3
     }
    }
   ],
   "accuracy": {
    "mean": 0.42,
    "min": 0.42,
    "max": 0.42
   },
   "accuracy_ci95": {
    "low": 0.32,
    "high": 0.52
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.2851344577114408,
    "min": 0.2783424025401473,
    "max": 0.2940311215352267
   },
   "p95_request_seconds": {
    "mean": 0.5217300009680912,
    "min": 0.48984210495837033,
    "max": 0.5403023459948599
   },
   "cost_per_million_questions": {
    "mean": 21.925,
    "min": 21.925,
    "max": 21.925
   },
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.6879,
    "min": 0.6879,
    "max": 0.6879
   },
   "sure_wrong": {
    "mean": 3,
    "min": 3,
    "max": 3
   },
   "sure": {
    "mean": 6,
    "min": 6,
    "max": 6
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.33,
     0.58,
     0.02,
     0.07
    ],
    "answers": 300,
    "yes_share": 0.45,
    "first_share_other": 0.15,
    "baseline": 0.38333986928104574,
    "above": 0.03666013071895424,
    "p": 0.2005547398612021,
    "first_in_10": 3,
    "confidence": {
     "answers": 300,
     "mean_right": 0.7095238095238096,
     "mean_wrong": 0.6722413793103448,
     "sure": 18,
     "sure_wrong": 9
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     1,
     1,
     1
    ],
    "hard_right_all_runs": 1
   }
  },
  {
   "provider": "jev",
   "model": "jev-1.14.0",
   "perplexity_mode": null,
   "request_model": "jev-1.14.0",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "TypeSafe's System One price",
    "source": "https://docs.typesafe.ai/models",
    "date": "2026-10-16",
    "input_per_million": 0.042,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T15:33:34+00:00",
     "ended": "2026-11-05T15:33:51+00:00",
     "questions": 100,
     "correct": 34,
     "accuracy": 0.34,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.16321340802824125,
     "p95_request_seconds": 0.19679734495002776,
     "input_tokens": 42884,
     "output_tokens": 3744,
     "cost_usd": 0.0018011280000000001,
     "cost_per_million_questions": 18.01128,
     "served_models": [
      "jev-1.14.0"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6638,
      "sure": 1,
      "sure_wrong": 1
     }
    },
    {
     "started": "2026-11-05T15:33:51+00:00",
     "ended": "2026-11-05T15:34:08+00:00",
     "questions": 100,
     "correct": 35,
     "accuracy": 0.35,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.16480575397145003,
     "p95_request_seconds": 0.2021742359502241,
     "input_tokens": 42884,
     "output_tokens": 3748,
     "cost_usd": 0.0018011280000000001,
     "cost_per_million_questions": 18.01128,
     "served_models": [
      "jev-1.14.0"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.663,
      "sure": 1,
      "sure_wrong": 1
     }
    },
    {
     "started": "2026-11-05T15:34:08+00:00",
     "ended": "2026-11-05T15:34:25+00:00",
     "questions": 100,
     "correct": 34,
     "accuracy": 0.34,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.1681436820072122,
     "p95_request_seconds": 0.20951338100712746,
     "input_tokens": 42884,
     "output_tokens": 3746,
     "cost_usd": 0.0018011280000000001,
     "cost_per_million_questions": 18.01128,
     "served_models": [
      "jev-1.14.0"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6594,
      "sure": 1,
      "sure_wrong": 1
     }
    }
   ],
   "accuracy": {
    "mean": 0.3433333333333333,
    "min": 0.34,
    "max": 0.35
   },
   "accuracy_ci95": {
    "low": 0.2533333333333333,
    "high": 0.43666666666666665
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.16538761466896781,
    "min": 0.16321340802824125,
    "max": 0.1681436820072122
   },
   "p95_request_seconds": {
    "mean": 0.20282832063579312,
    "min": 0.19679734495002776,
    "max": 0.20951338100712746
   },
   "cost_per_million_questions": {
    "mean": 18.01128,
    "min": 18.01128,
    "max": 18.01128
   },
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.6620666666666667,
    "min": 0.6594,
    "max": 0.6638
   },
   "sure_wrong": {
    "mean": 1,
    "min": 1,
    "max": 1
   },
   "sure": {
    "mean": 1,
    "min": 1,
    "max": 1
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.2833333333333333,
     0.5833333333333334,
     0.06,
     0.07333333333333333
    ],
    "answers": 300,
    "yes_share": 0.3333333333333333,
    "first_share_other": 0.20833333333333334,
    "baseline": 0.371359477124183,
    "above": -0.028026143790849667,
    "p": 0.7314446647589111,
    "first_in_10": 3,
    "confidence": {
     "answers": 300,
     "mean_right": 0.6529126213592233,
     "mean_wrong": 0.6668527918781726,
     "sure": 3,
     "sure_wrong": 3
    },
    "p_holm": 1.0,
    "explained": true,
    "hard_right": [
     0,
     0,
     0
    ],
    "hard_right_all_runs": 0
   }
  },
  {
   "provider": "jev",
   "model": "jev-1.13.0",
   "perplexity_mode": null,
   "request_model": "jev-1.13.0",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "TypeSafe's System One price",
    "source": "https://typesafe.ai/blog/introducing-system-one-models-and-jev",
    "date": "2026-10-08",
    "input_per_million": 0.042,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T15:34:26+00:00",
     "ended": "2026-11-05T15:34:44+00:00",
     "questions": 100,
     "correct": 34,
     "accuracy": 0.34,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.1684987290063873,
     "p95_request_seconds": 0.25876956805586815,
     "input_tokens": 42884,
     "output_tokens": 3747,
     "cost_usd": 0.0018011280000000001,
     "cost_per_million_questions": 18.01128,
     "served_models": [
      "jev-1.13.0"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6900000000000001,
      "sure": 1,
      "sure_wrong": 1
     }
    },
    {
     "started": "2026-11-05T15:34:44+00:00",
     "ended": "2026-11-05T15:35:02+00:00",
     "questions": 100,
     "correct": 36,
     "accuracy": 0.36,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.1736479609971866,
     "p95_request_seconds": 0.24512670503463596,
     "input_tokens": 42884,
     "output_tokens": 3748,
     "cost_usd": 0.0018011280000000001,
     "cost_per_million_questions": 18.01128,
     "served_models": [
      "jev-1.13.0"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6916,
      "sure": 1,
      "sure_wrong": 1
     }
    },
    {
     "started": "2026-11-05T15:35:02+00:00",
     "ended": "2026-11-05T15:35:20+00:00",
     "questions": 100,
     "correct": 35,
     "accuracy": 0.35,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.1670556865283288,
     "p95_request_seconds": 0.22127021895721555,
     "input_tokens": 42884,
     "output_tokens": 3746,
     "cost_usd": 0.0018011280000000001,
     "cost_per_million_questions": 18.01128,
     "served_models": [
      "jev-1.13.0"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6932,
      "sure": 1,
      "sure_wrong": 1
     }
    }
   ],
   "accuracy": {
    "mean": 0.35,
    "min": 0.34,
    "max": 0.36
   },
   "accuracy_ci95": {
    "low": 0.25666666666666665,
    "high": 0.44666666666666666
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.16973412551063424,
    "min": 0.1670556865283288,
    "max": 0.1736479609971866
   },
   "p95_request_seconds": {
    "mean": 0.24172216401590654,
    "min": 0.22127021895721555,
    "max": 0.25876956805586815
   },
   "cost_per_million_questions": {
    "mean": 18.01128,
    "min": 18.01128,
    "max": 18.01128
   },
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.6916,
    "min": 0.6900000000000001,
    "max": 0.6932
   },
   "sure_wrong": {
    "mean": 1,
    "min": 1,
    "max": 1
   },
   "sure": {
    "mean": 1,
    "min": 1,
    "max": 1
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.30333333333333334,
     0.5766666666666667,
     0.04666666666666667,
     0.07333333333333333
    ],
    "answers": 300,
    "yes_share": 0.35555555555555557,
    "first_share_other": 0.225,
    "baseline": 0.37090849673202614,
    "above": -0.02090849673202616,
    "p": 0.6780287302096286,
    "first_in_10": 3,
    "confidence": {
     "answers": 300,
     "mean_right": 0.6662857142857143,
     "mean_wrong": 0.7052307692307692,
     "sure": 3,
     "sure_wrong": 3
    },
    "p_holm": 1.0,
    "explained": true,
    "hard_right": [
     0,
     0,
     0
    ],
    "hard_right_all_runs": 0
   }
  },
  {
   "provider": "liquid",
   "model": "d1-ultra",
   "perplexity_mode": null,
   "request_model": "d1-ultra",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "Liquid AI's d1-ultra price",
    "source": "https://www.liquid.ai/blog/d1-ultra",
    "date": "2026-10-16",
    "input_per_million": 0.042,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T15:33:34+00:00",
     "ended": "2026-11-05T15:34:01+00:00",
     "questions": 100,
     "correct": 58,
     "accuracy": 0.58,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.22622280550422147,
     "p95_request_seconds": 0.5500040600309148,
     "input_tokens": 14696,
     "output_tokens": 0,
     "cost_usd": 0.000617232,
     "cost_per_million_questions": 6.17232,
     "served_models": [
      "d1-ultra"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6804643123387472,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:34:02+00:00",
     "ended": "2026-11-05T15:34:29+00:00",
     "questions": 100,
     "correct": 59,
     "accuracy": 0.59,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.21746412897482514,
     "p95_request_seconds": 0.5672525530681014,
     "input_tokens": 14696,
     "output_tokens": 0,
     "cost_usd": 0.000617232,
     "cost_per_million_questions": 6.17232,
     "served_models": [
      "d1-ultra"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6741317367235411,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:34:29+00:00",
     "ended": "2026-11-05T15:34:57+00:00",
     "questions": 100,
     "correct": 56,
     "accuracy": 0.56,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.21613645704928786,
     "p95_request_seconds": 0.5626515010371804,
     "input_tokens": 14696,
     "output_tokens": 0,
     "cost_usd": 0.000617232,
     "cost_per_million_questions": 6.17232,
     "served_models": [
      "d1-ultra"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6714588762953762,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.5766666666666667,
    "min": 0.56,
    "max": 0.59
   },
   "accuracy_ci95": {
    "low": 0.48333333333333334,
    "high": 0.6666666666666667
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.21994113050944483,
    "min": 0.21613645704928786,
    "max": 0.22622280550422147
   },
   "p95_request_seconds": {
    "mean": 0.5599693713787323,
    "min": 0.5500040600309148,
    "max": 0.5672525530681014
   },
   "cost_per_million_questions": {
    "mean": 6.17232,
    "min": 6.17232,
    "max": 6.17232
   },
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.6753516417858881,
    "min": 0.6714588762953762,
    "max": 0.6804643123387472
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.53,
     0.39,
     0.02,
     0.06
    ],
    "answers": 300,
    "yes_share": 0.5777777777777777,
    "first_share_other": 0.4583333333333333,
    "baseline": 0.47635294117647053,
    "above": 0.10031372549019613,
    "p": 0.018401860957492744,
    "first_in_10": 5,
    "confidence": {
     "answers": 300,
     "mean_right": 0.6980485858649781,
     "mean_wrong": 0.6444337573316947,
     "sure": 0,
     "sure_wrong": 0
    },
    "reversed": {
     "run_1_accuracy": 0.58,
     "accuracy": 0.54,
     "change": -0.04,
     "same_choice_as_run_1": 0.86,
     "runs_1_2_agree": 0.89,
     "errors": 0,
     "missing": 0
    },
    "p_holm": 0.6624669944697388,
    "explained": false,
    "hard_right": [
     1,
     1,
     1
    ],
    "hard_right_all_runs": 1
   }
  },
  {
   "provider": "liquid",
   "model": "d1.1",
   "perplexity_mode": null,
   "request_model": "d1.1",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "Liquid AI's price for d1.1, as its API bills it",
    "source": "https://www.liquid.ai/pricing",
    "date": "2026-10-24",
    "input_per_million": 0.02,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T15:34:57+00:00",
     "ended": "2026-11-05T15:35:23+00:00",
     "questions": 100,
     "correct": 52,
     "accuracy": 0.52,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.184678494988475,
     "p95_request_seconds": 0.49098465708084404,
     "input_tokens": 14696,
     "output_tokens": 0,
     "cost_usd": 0.00029392000000000003,
     "cost_per_million_questions": 2.9392,
     "served_models": [
      "d1.1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6755529330394516,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:35:23+00:00",
     "ended": "2026-11-05T15:35:43+00:00",
     "questions": 100,
     "correct": 51,
     "accuracy": 0.51,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.17807031649863347,
     "p95_request_seconds": 0.4398084559943527,
     "input_tokens": 14696,
     "output_tokens": 0,
     "cost_usd": 0.00029392000000000003,
     "cost_per_million_questions": 2.9392,
     "served_models": [
      "d1.1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6568720471836241,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:35:43+00:00",
     "ended": "2026-11-05T15:36:03+00:00",
     "questions": 100,
     "correct": 56,
     "accuracy": 0.56,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.17693206446710974,
     "p95_request_seconds": 0.31294334400445223,
     "input_tokens": 14696,
     "output_tokens": 0,
     "cost_usd": 0.00029392000000000003,
     "cost_per_million_questions": 2.9392,
     "served_models": [
      "d1.1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6607066307222138,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.53,
    "min": 0.51,
    "max": 0.56
   },
   "accuracy_ci95": {
    "low": 0.44,
    "high": 0.62
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.17989362531807274,
    "min": 0.17693206446710974,
    "max": 0.184678494988475
   },
   "p95_request_seconds": {
    "mean": 0.41457881902654964,
    "min": 0.31294334400445223,
    "max": 0.49098465708084404
   },
   "cost_per_million_questions": {
    "mean": 2.9392,
    "min": 2.9392,
    "max": 2.9392
   },
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.6643772036484299,
    "min": 0.6568720471836241,
    "max": 0.6755529330394516
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.6266666666666667,
     0.2866666666666667,
     0.06,
     0.02666666666666667
    ],
    "answers": 300,
    "yes_share": 0.6611111111111111,
    "first_share_other": 0.575,
    "baseline": 0.5315272331154685,
    "above": -0.0015272331154684293,
    "p": 0.5132292061719095,
    "first_in_10": 6,
    "confidence": {
     "answers": 300,
     "mean_right": 0.6833261959408062,
     "mean_wrong": 0.6430091910634096,
     "sure": 0,
     "sure_wrong": 0
    },
    "p_holm": 1.0,
    "explained": true,
    "hard_right": [
     2,
     1,
     1
    ],
    "hard_right_all_runs": 1
   }
  },
  {
   "provider": "liquid",
   "model": "d1:free",
   "perplexity_mode": null,
   "request_model": "d1",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "Liquid's list price for the same model",
    "source": "https://www.liquid.ai/pricing",
    "date": "2026-10-18",
    "input_per_million": 0.02,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T15:36:03+00:00",
     "ended": "2026-11-05T15:36:25+00:00",
     "questions": 100,
     "correct": 49,
     "accuracy": 0.49,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.18305583449546248,
     "p95_request_seconds": 0.4564928689505905,
     "input_tokens": 14696,
     "output_tokens": 0,
     "cost_usd": 0.00029392000000000003,
     "cost_per_million_questions": 2.9392,
     "served_models": [
      "d1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6822068694761986,
      "sure": 2,
      "sure_wrong": 1
     }
    },
    {
     "started": "2026-11-05T15:36:25+00:00",
     "ended": "2026-11-05T15:36:47+00:00",
     "questions": 100,
     "correct": 47,
     "accuracy": 0.47,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.1851450459798798,
     "p95_request_seconds": 0.46770492393989116,
     "input_tokens": 14696,
     "output_tokens": 0,
     "cost_usd": 0.00029392000000000003,
     "cost_per_million_questions": 2.9392,
     "served_models": [
      "d1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6891535587431786,
      "sure": 1,
      "sure_wrong": 1
     }
    },
    {
     "started": "2026-11-05T15:36:48+00:00",
     "ended": "2026-11-05T15:37:09+00:00",
     "questions": 100,
     "correct": 47,
     "accuracy": 0.47,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.18636794644407928,
     "p95_request_seconds": 0.3497232840163633,
     "input_tokens": 14696,
     "output_tokens": 0,
     "cost_usd": 0.00029392000000000003,
     "cost_per_million_questions": 2.9392,
     "served_models": [
      "d1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6929214975452629,
      "sure": 1,
      "sure_wrong": 1
     }
    }
   ],
   "accuracy": {
    "mean": 0.4766666666666666,
    "min": 0.47,
    "max": 0.49
   },
   "accuracy_ci95": {
    "low": 0.38666666666666666,
    "high": 0.5666666666666667
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.1848562756398072,
    "min": 0.18305583449546248,
    "max": 0.18636794644407928
   },
   "p95_request_seconds": {
    "mean": 0.4246403589689483,
    "min": 0.3497232840163633,
    "max": 0.46770492393989116
   },
   "cost_per_million_questions": {
    "mean": 2.9392,
    "min": 2.9392,
    "max": 2.9392
   },
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.68809397525488,
    "min": 0.6822068694761986,
    "max": 0.6929214975452629
   },
   "sure_wrong": {
    "mean": 1,
    "min": 1,
    "max": 1
   },
   "sure": {
    "mean": 1.3333333333333333,
    "min": 1,
    "max": 2
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.48,
     0.43333333333333335,
     0.056666666666666664,
     0.03
    ],
    "answers": 300,
    "yes_share": 0.5055555555555555,
    "first_share_other": 0.44166666666666665,
    "baseline": 0.462359477124183,
    "above": 0.014307189542483612,
    "p": 0.3848052896424542,
    "first_in_10": 5,
    "confidence": {
     "answers": 300,
     "mean_right": 0.7025888471102542,
     "mean_wrong": 0.6748916397432972,
     "sure": 4,
     "sure_wrong": 3
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     0,
     0,
     0
    ],
    "hard_right_all_runs": 0
   }
  },
  {
   "provider": "celeris",
   "model": "celeris-1-decision",
   "perplexity_mode": null,
   "request_model": "celeris-1-decision",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "Celeris's decision model price",
    "source": "https://docs.celeris.ai/pricing",
    "date": "2026-10-10",
    "input_per_million": 0.04,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T15:33:34+00:00",
     "ended": "2026-11-05T15:34:12+00:00",
     "questions": 100,
     "correct": 34,
     "accuracy": 0.34,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.38275510648963973,
     "p95_request_seconds": 0.38867232692427933,
     "input_tokens": 21153,
     "output_tokens": 0,
     "cost_usd": 0.00084612,
     "cost_per_million_questions": 8.4612,
     "served_models": [
      "celeris-1-decision"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.692,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:34:13+00:00",
     "ended": "2026-11-05T15:34:51+00:00",
     "questions": 100,
     "correct": 33,
     "accuracy": 0.33,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.38386286300374195,
     "p95_request_seconds": 0.3887620309833437,
     "input_tokens": 21153,
     "output_tokens": 0,
     "cost_usd": 0.00084612,
     "cost_per_million_questions": 8.4612,
     "served_models": [
      "celeris-1-decision"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6926,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:34:51+00:00",
     "ended": "2026-11-05T15:35:29+00:00",
     "questions": 100,
     "correct": 33,
     "accuracy": 0.33,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.3842481910251081,
     "p95_request_seconds": 0.389283207943663,
     "input_tokens": 21153,
     "output_tokens": 0,
     "cost_usd": 0.00084612,
     "cost_per_million_questions": 8.4612,
     "served_models": [
      "celeris-1-decision"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6933,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.33333333333333337,
    "min": 0.33,
    "max": 0.34
   },
   "accuracy_ci95": {
    "low": 0.24333333333333332,
    "high": 0.42666666666666664
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.38362205350616324,
    "min": 0.38275510648963973,
    "max": 0.3842481910251081
   },
   "p95_request_seconds": {
    "mean": 0.38890585528376204,
    "min": 0.38867232692427933,
    "max": 0.389283207943663
   },
   "cost_per_million_questions": {
    "mean": 8.4612,
    "min": 8.4612,
    "max": 8.4612
   },
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.6926333333333333,
    "min": 0.692,
    "max": 0.6933
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.18,
     0.72,
     0.03,
     0.07
    ],
    "answers": 300,
    "yes_share": 0.16666666666666666,
    "first_share_other": 0.2,
    "baseline": 0.31420915032679736,
    "above": 0.01912418300653601,
    "p": 0.31299842235989306,
    "first_in_10": 2,
    "confidence": {
     "answers": 300,
     "mean_right": 0.7005,
     "mean_wrong": 0.6887,
     "sure": 0,
     "sure_wrong": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     0,
     0,
     0
    ],
    "hard_right_all_runs": 0
   }
  },
  {
   "provider": "subq",
   "model": "subq-snap-1.1",
   "perplexity_mode": null,
   "request_model": "subq-snap-1.1",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "SubQ's list price",
    "source": "https://docs.subq.ai/using-the-api/models/",
    "date": "2026-11-01",
    "input_per_million": 0.04,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T15:33:34+00:00",
     "ended": "2026-11-05T15:34:12+00:00",
     "questions": 100,
     "correct": 53,
     "accuracy": 0.53,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.37701621698215604,
     "p95_request_seconds": 0.3913654339266941,
     "input_tokens": 19838,
     "output_tokens": 0,
     "cost_usd": 0.00079352,
     "cost_per_million_questions": 7.9352,
     "served_models": [
      "subq-snap-1.1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7024527570605278,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:34:12+00:00",
     "ended": "2026-11-05T15:34:50+00:00",
     "questions": 100,
     "correct": 53,
     "accuracy": 0.53,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.3769398934673518,
     "p95_request_seconds": 0.3865992579376325,
     "input_tokens": 19838,
     "output_tokens": 0,
     "cost_usd": 0.00079352,
     "cost_per_million_questions": 7.9352,
     "served_models": [
      "subq-snap-1.1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7024527570605278,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:34:50+00:00",
     "ended": "2026-11-05T15:35:28+00:00",
     "questions": 100,
     "correct": 53,
     "accuracy": 0.53,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.3785566790611483,
     "p95_request_seconds": 0.3933416190557182,
     "input_tokens": 19838,
     "output_tokens": 0,
     "cost_usd": 0.00079352,
     "cost_per_million_questions": 7.9352,
     "served_models": [
      "subq-snap-1.1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7024527570605278,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.53,
    "min": 0.53,
    "max": 0.53
   },
   "accuracy_ci95": {
    "low": 0.43,
    "high": 0.63
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.37750426317021873,
    "min": 0.3769398934673518,
    "max": 0.3785566790611483
   },
   "p95_request_seconds": {
    "mean": 0.39043543697334826,
    "min": 0.3865992579376325,
    "max": 0.3933416190557182
   },
   "cost_per_million_questions": {
    "mean": 7.9352,
    "min": 7.9352,
    "max": 7.9352
   },
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.7024527570605278,
    "min": 0.7024527570605278,
    "max": 0.7024527570605278
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.48,
     0.41,
     0.08,
     0.03
    ],
    "answers": 300,
    "yes_share": 0.6,
    "first_share_other": 0.3,
    "baseline": 0.48026143790849674,
    "above": 0.04973856209150329,
    "p": 0.14689579456130697,
    "first_in_10": 5,
    "confidence": {
     "answers": 300,
     "mean_right": 0.7538855761851905,
     "mean_wrong": 0.6444540461327167,
     "sure": 0,
     "sure_wrong": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     0,
     0,
     0
    ],
    "hard_right_all_runs": 0
   }
  },
  {
   "provider": "subq",
   "model": "subq-snap-preview",
   "perplexity_mode": null,
   "request_model": "subq-snap-preview",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "SubQ's Decisions API price",
    "source": "https://docs.subq.ai/using-the-api/models/",
    "date": "2026-10-13",
    "input_per_million": 0.05,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T15:35:28+00:00",
     "ended": "2026-11-05T15:36:06+00:00",
     "questions": 100,
     "correct": 53,
     "accuracy": 0.53,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.3793836410040967,
     "p95_request_seconds": 0.3928555540041998,
     "input_tokens": 19838,
     "output_tokens": 0,
     "cost_usd": 0.0009919,
     "cost_per_million_questions": 9.918999999999999,
     "served_models": [
      "subq-snap-preview"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7024527570605278,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:36:06+00:00",
     "ended": "2026-11-05T15:36:44+00:00",
     "questions": 100,
     "correct": 53,
     "accuracy": 0.53,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.3814237375045195,
     "p95_request_seconds": 0.3926281299209222,
     "input_tokens": 19838,
     "output_tokens": 0,
     "cost_usd": 0.0009919,
     "cost_per_million_questions": 9.918999999999999,
     "served_models": [
      "subq-snap-preview"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7024527570605278,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:36:44+00:00",
     "ended": "2026-11-05T15:37:22+00:00",
     "questions": 100,
     "correct": 53,
     "accuracy": 0.53,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.3792807099525817,
     "p95_request_seconds": 0.3914589410414919,
     "input_tokens": 19838,
     "output_tokens": 0,
     "cost_usd": 0.0009919,
     "cost_per_million_questions": 9.918999999999999,
     "served_models": [
      "subq-snap-preview"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7024527570605278,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.53,
    "min": 0.53,
    "max": 0.53
   },
   "accuracy_ci95": {
    "low": 0.43,
    "high": 0.63
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.3800293628203993,
    "min": 0.3792807099525817,
    "max": 0.3814237375045195
   },
   "p95_request_seconds": {
    "mean": 0.39231420832220465,
    "min": 0.3914589410414919,
    "max": 0.3928555540041998
   },
   "cost_per_million_questions": {
    "mean": 9.918999999999999,
    "min": 9.918999999999999,
    "max": 9.918999999999999
   },
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.7024527570605278,
    "min": 0.7024527570605278,
    "max": 0.7024527570605278
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.48,
     0.41,
     0.08,
     0.03
    ],
    "answers": 300,
    "yes_share": 0.6,
    "first_share_other": 0.3,
    "baseline": 0.48026143790849674,
    "above": 0.04973856209150329,
    "p": 0.14689579456130697,
    "first_in_10": 5,
    "confidence": {
     "answers": 300,
     "mean_right": 0.7538855761851905,
     "mean_wrong": 0.6444540461327167,
     "sure": 0,
     "sure_wrong": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     0,
     0,
     0
    ],
    "hard_right_all_runs": 0
   }
  },
  {
   "provider": "meragpt",
   "model": "decider-1",
   "perplexity_mode": null,
   "request_model": "decider-1",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "meraGPT's Decider 1 price",
    "source": "https://meragpt.com/docs/models",
    "date": "2026-10-19",
    "input_per_million": 0.04,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T15:33:34+00:00",
     "ended": "2026-11-05T15:34:42+00:00",
     "questions": 100,
     "correct": 50,
     "accuracy": 0.5,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.6351454495452344,
     "p95_request_seconds": 0.6837720080511644,
     "input_tokens": 17025,
     "output_tokens": 0,
     "cost_usd": 0.000681,
     "cost_per_million_questions": 6.81,
     "served_models": [
      "decider-1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.60308,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:34:43+00:00",
     "ended": "2026-11-05T15:35:46+00:00",
     "questions": 100,
     "correct": 50,
     "accuracy": 0.5,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.6281938590109348,
     "p95_request_seconds": 0.706447319011204,
     "input_tokens": 17025,
     "output_tokens": 0,
     "cost_usd": 0.000681,
     "cost_per_million_questions": 6.81,
     "served_models": [
      "decider-1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.60308,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:35:47+00:00",
     "ended": "2026-11-05T15:36:49+00:00",
     "questions": 100,
     "correct": 50,
     "accuracy": 0.5,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.6244177459739149,
     "p95_request_seconds": 0.6799519609194249,
     "input_tokens": 17025,
     "output_tokens": 0,
     "cost_usd": 0.000681,
     "cost_per_million_questions": 6.81,
     "served_models": [
      "decider-1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.60308,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.5,
    "min": 0.5,
    "max": 0.5
   },
   "accuracy_ci95": {
    "low": 0.4,
    "high": 0.6
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.629252351510028,
    "min": 0.6244177459739149,
    "max": 0.6351454495452344
   },
   "p95_request_seconds": {
    "mean": 0.6900570959939311,
    "min": 0.6799519609194249,
    "max": 0.706447319011204
   },
   "cost_per_million_questions": {
    "mean": 6.81,
    "min": 6.81,
    "max": 6.81
   },
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.60308,
    "min": 0.60308,
    "max": 0.60308
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.46,
     0.48,
     0.05,
     0.01
    ],
    "answers": 300,
    "yes_share": 0.5333333333333333,
    "first_share_other": 0.35,
    "baseline": 0.47237908496732023,
    "above": 0.02762091503267977,
    "p": 0.2859640200651665,
    "first_in_10": 5,
    "confidence": {
     "answers": 300,
     "mean_right": 0.613346,
     "mean_wrong": 0.592814,
     "sure": 0,
     "sure_wrong": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     5,
     5,
     5
    ],
    "hard_right_all_runs": 5
   }
  },
  {
   "provider": "nace",
   "model": "drex-max-v1.1",
   "perplexity_mode": null,
   "request_model": "drex-max-v1.1",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "Nace.AI's list price (launch price $0.03 per 1M input tokens, 50% off, is what Nace bills)",
    "source": "https://nace.ai/drex",
    "date": "2026-10-30",
    "input_per_million": 0.06,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T15:33:34+00:00",
     "ended": "2026-11-05T15:34:12+00:00",
     "questions": 100,
     "correct": 48,
     "accuracy": 0.48,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.33156991348369047,
     "p95_request_seconds": 0.4187116810353473,
     "input_tokens": 19838,
     "output_tokens": 10153,
     "cost_usd": 0.00119028,
     "cost_per_million_questions": 11.9028,
     "served_models": [
      "drex-max-v1.1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7162192470805888,
      "sure": 1,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:34:13+00:00",
     "ended": "2026-11-05T15:35:07+00:00",
     "questions": 100,
     "correct": 48,
     "accuracy": 0.48,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 4,
     "median_request_seconds": 0.331321606063284,
     "p95_request_seconds": 0.5107417199760675,
     "input_tokens": 19838,
     "output_tokens": 10152,
     "cost_usd": 0.00119028,
     "cost_per_million_questions": 11.9028,
     "served_models": [
      "drex-max-v1.1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7170888714969105,
      "sure": 1,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:35:07+00:00",
     "ended": "2026-11-05T15:36:00+00:00",
     "questions": 100,
     "correct": 47,
     "accuracy": 0.47,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 4,
     "median_request_seconds": 0.33045578602468595,
     "p95_request_seconds": 0.47940575797110796,
     "input_tokens": 19838,
     "output_tokens": 10152,
     "cost_usd": 0.00119028,
     "cost_per_million_questions": 11.9028,
     "served_models": [
      "drex-max-v1.1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7170255166793795,
      "sure": 1,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.4766666666666666,
    "min": 0.47,
    "max": 0.48
   },
   "accuracy_ci95": {
    "low": 0.38,
    "high": 0.5733333333333334
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.3311157685238868,
    "min": 0.33045578602468595,
    "max": 0.33156991348369047
   },
   "p95_request_seconds": {
    "mean": 0.46961971966084093,
    "min": 0.4187116810353473,
    "max": 0.5107417199760675
   },
   "cost_per_million_questions": {
    "mean": 11.9028,
    "min": 11.9028,
    "max": 11.9028
   },
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.7167778784189596,
    "min": 0.7162192470805888,
    "max": 0.7170888714969105
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 1,
    "min": 1,
    "max": 1
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.38333333333333336,
     0.5466666666666666,
     0.04,
     0.03
    ],
    "answers": 300,
    "yes_share": 0.45555555555555555,
    "first_share_other": 0.275,
    "baseline": 0.4335816993464052,
    "above": 0.0430849673202614,
    "p": 0.18019968896491712,
    "first_in_10": 4,
    "confidence": {
     "answers": 300,
     "mean_right": 0.7192796881321902,
     "mean_wrong": 0.7144991600177368,
     "sure": 3,
     "sure_wrong": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     1,
     1,
     1
    ],
    "hard_right_all_runs": 1
   }
  },
  {
   "provider": "nace",
   "model": "drex-v2.0",
   "perplexity_mode": null,
   "request_model": "drex-v2.0",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "Nace.AI's list price",
    "source": "https://console.nace.ai/docs/drex/reference/pricing",
    "date": "2026-10-30",
    "input_per_million": 0.06,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T15:36:00+00:00",
     "ended": "2026-11-05T15:36:50+00:00",
     "questions": 100,
     "correct": 31,
     "accuracy": 0.31,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 6,
     "median_request_seconds": 0.2944185205269605,
     "p95_request_seconds": 1.4998827240196988,
     "input_tokens": 13514,
     "output_tokens": 5926,
     "cost_usd": 0.00081084,
     "cost_per_million_questions": 8.1084,
     "served_models": [
      "drex-v2.0"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.685149,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:36:50+00:00",
     "ended": "2026-11-05T15:37:42+00:00",
     "questions": 100,
     "correct": 31,
     "accuracy": 0.31,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 7,
     "median_request_seconds": 0.2919942224980332,
     "p95_request_seconds": 1.512011977029033,
     "input_tokens": 13514,
     "output_tokens": 5926,
     "cost_usd": 0.00081084,
     "cost_per_million_questions": 8.1084,
     "served_models": [
      "drex-v2.0"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.685149,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:37:43+00:00",
     "ended": "2026-11-05T15:38:23+00:00",
     "questions": 100,
     "correct": 31,
     "accuracy": 0.31,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 8,
     "median_request_seconds": 0.2915091179893352,
     "p95_request_seconds": 1.5087617490207776,
     "input_tokens": 13514,
     "output_tokens": 5926,
     "cost_usd": 0.00081084,
     "cost_per_million_questions": 8.1084,
     "served_models": [
      "drex-v2.0"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.685149,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.31,
    "min": 0.31,
    "max": 0.31
   },
   "accuracy_ci95": {
    "low": 0.22,
    "high": 0.4
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.2926406203381096,
    "min": 0.2915091179893352,
    "max": 0.2944185205269605
   },
   "p95_request_seconds": {
    "mean": 1.506885483356503,
    "min": 1.4998827240196988,
    "max": 1.512011977029033
   },
   "cost_per_million_questions": {
    "mean": 8.1084,
    "min": 8.1084,
    "max": 8.1084
   },
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.685149,
    "min": 0.685149,
    "max": 0.685149
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.19,
     0.72,
     0.05,
     0.04
    ],
    "answers": 300,
    "yes_share": 0.18333333333333332,
    "first_share_other": 0.2,
    "baseline": 0.3174575163398693,
    "above": -0.007457516339869297,
    "p": 0.5755040214683469,
    "first_in_10": 2,
    "confidence": {
     "answers": 300,
     "mean_right": 0.6763741935483871,
     "mean_wrong": 0.6890913043478261,
     "sure": 0,
     "sure_wrong": 0
    },
    "p_holm": 1.0,
    "explained": true,
    "hard_right": [
     0,
     0,
     0
    ],
    "hard_right_all_runs": 0
   }
  },
  {
   "provider": "openrouter",
   "model": "microsoft/microsoft-decision-1",
   "perplexity_mode": null,
   "request_model": "microsoft/microsoft-decision-1",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "OpenRouter's Microsoft-Decision-1 price",
    "source": "https://openrouter.ai/microsoft/microsoft-decision-1",
    "date": "2026-10-17",
    "input_per_million": 0.042,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T15:33:34+00:00",
     "ended": "2026-11-05T15:34:08+00:00",
     "questions": 100,
     "correct": 49,
     "accuracy": 0.49,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.3188218059949577,
     "p95_request_seconds": 0.3918567489599809,
     "input_tokens": 14537,
     "output_tokens": 100,
     "cost_usd": 0.000610554,
     "cost_per_million_questions": 6.10554,
     "served_models": [
      "microsoft/microsoft-decision-1-20261009"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6587888362496626,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:34:08+00:00",
     "ended": "2026-11-05T15:34:41+00:00",
     "questions": 100,
     "correct": 49,
     "accuracy": 0.49,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.3179865350248292,
     "p95_request_seconds": 0.38290938502177596,
     "input_tokens": 14537,
     "output_tokens": 100,
     "cost_usd": 0.000610554,
     "cost_per_million_questions": 6.10554,
     "served_models": [
      "microsoft/microsoft-decision-1-20261009"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6649398559180528,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T15:34:41+00:00",
     "ended": "2026-11-05T15:35:13+00:00",
     "questions": 100,
     "correct": 49,
     "accuracy": 0.49,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.31402686901856214,
     "p95_request_seconds": 0.37895343999844044,
     "input_tokens": 14537,
     "output_tokens": 100,
     "cost_usd": 0.000610554,
     "cost_per_million_questions": 6.10554,
     "served_models": [
      "microsoft/microsoft-decision-1-20261009"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6588867914717088,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.49,
    "min": 0.49,
    "max": 0.49
   },
   "accuracy_ci95": {
    "low": 0.39333333333333337,
    "high": 0.5866666666666667
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.316945070012783,
    "min": 0.31402686901856214,
    "max": 0.3188218059949577
   },
   "p95_request_seconds": {
    "mean": 0.38457319132673246,
    "min": 0.37895343999844044,
    "max": 0.3918567489599809
   },
   "cost_per_million_questions": {
    "mean": 6.10554,
    "min": 6.10554,
    "max": 6.10554
   },
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.660871827879808,
    "min": 0.6587888362496626,
    "max": 0.6649398559180528
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.4066666666666667,
     0.5433333333333333,
     0.03,
     0.02
    ],
    "answers": 300,
    "yes_share": 0.4111111111111111,
    "first_share_other": 0.4,
    "baseline": 0.44026797385620914,
    "above": 0.04973202614379085,
    "p": 0.14955853549833387,
    "first_in_10": 4,
    "confidence": {
     "answers": 300,
     "mean_right": 0.6730279215211508,
     "mean_wrong": 0.649192443793028,
     "sure": 0,
     "sure_wrong": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     1,
     0,
     1
    ],
    "hard_right_all_runs": 0
   }
  },
  {
   "provider": "openrouter",
   "model": "inception/mercury-decide",
   "perplexity_mode": null,
   "request_model": "inception/mercury-decide",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "OpenRouter's Mercury Decide price",
    "source": "https://openrouter.ai/inception/mercury-decide",
    "date": "2026-10-17",
    "input_per_million": 0.02,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T15:35:14+00:00",
     "ended": "2026-11-05T15:35:44+00:00",
     "questions": 100,
     "correct": 54,
     "accuracy": 0.54,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.3009369229548611,
     "p95_request_seconds": 0.36108742805663496,
     "input_tokens": 17870,
     "output_tokens": 100,
     "cost_usd": 0.0003574,
     "cost_per_million_questions": 3.574,
     "served_models": [
      "inception/mercury-decide-20260930"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6489074305373813,
      "sure": 2,
      "sure_wrong": 1
     }
    },
    {
     "started": "2026-11-05T15:35:44+00:00",
     "ended": "2026-11-05T15:36:14+00:00",
     "questions": 100,
     "correct": 53,
     "accuracy": 0.53,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.2840292659820989,
     "p95_request_seconds": 0.3550228120293468,
     "input_tokens": 17827,
     "output_tokens": 100,
     "cost_usd": 0.00035654,
     "cost_per_million_questions": 3.5654000000000003,
     "served_models": [
      "inception/mercury-decide-20260930"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6545953282629683,
      "sure": 2,
      "sure_wrong": 1
     }
    },
    {
     "started": "2026-11-05T15:36:14+00:00",
     "ended": "2026-11-05T15:36:43+00:00",
     "questions": 100,
     "correct": 53,
     "accuracy": 0.53,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.2880422624293715,
     "p95_request_seconds": 0.3780156910652295,
     "input_tokens": 17898,
     "output_tokens": 100,
     "cost_usd": 0.00035796,
     "cost_per_million_questions": 3.5796,
     "served_models": [
      "inception/mercury-decide-20260930"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6470775766114888,
      "sure": 2,
      "sure_wrong": 1
     }
    }
   ],
   "accuracy": {
    "mean": 0.5333333333333333,
    "min": 0.53,
    "max": 0.54
   },
   "accuracy_ci95": {
    "low": 0.44,
    "high": 0.6266666666666666
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.29100281712211046,
    "min": 0.2840292659820989,
    "max": 0.3009369229548611
   },
   "p95_request_seconds": {
    "mean": 0.3647086437170704,
    "min": 0.3550228120293468,
    "max": 0.3780156910652295
   },
   "cost_per_million_questions": {
    "mean": 3.573,
    "min": 3.5654000000000003,
    "max": 3.5796
   },
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.6501934451372795,
    "min": 0.6470775766114888,
    "max": 0.6545953282629683
   },
   "sure_wrong": {
    "mean": 1,
    "min": 1,
    "max": 1
   },
   "sure": {
    "mean": 2,
    "min": 2,
    "max": 2
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.5166666666666667,
     0.38333333333333336,
     0.06,
     0.04
    ],
    "answers": 300,
    "yes_share": 0.5611111111111111,
    "first_share_other": 0.45,
    "baseline": 0.4833202614379085,
    "above": 0.05001307189542481,
    "p": 0.1496356181032994,
    "first_in_10": 5,
    "confidence": {
     "answers": 300,
     "mean_right": 0.6822812391306184,
     "mean_wrong": 0.6135216805734636,
     "sure": 6,
     "sure_wrong": 3
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     3,
     2,
     2
    ],
    "hard_right_all_runs": 1
   }
  },
  {
   "provider": "baseten",
   "model": "decision-models/baseten-mammoth-27b-latest",
   "perplexity_mode": null,
   "request_model": "decision-models/baseten-mammoth-27b-latest",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "Baseten's Model API list price",
    "source": "https://www.baseten.co/library/mammoth-1/",
    "date": "2026-11-01",
    "input_per_million": 0.1,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T16:24:10+00:00",
     "ended": "2026-11-05T16:24:37+00:00",
     "questions": 100,
     "correct": 37,
     "accuracy": 0.37,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.18496603652602062,
     "p95_request_seconds": 0.3544489030027762,
     "input_tokens": 20420,
     "output_tokens": 100,
     "cost_usd": 0.002042,
     "cost_per_million_questions": 20.42,
     "served_models": [
      "decision-models/baseten-mammoth-27b-latest"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6994548448408199,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:24:37+00:00",
     "ended": "2026-11-05T16:24:56+00:00",
     "questions": 100,
     "correct": 37,
     "accuracy": 0.37,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.18387077003717422,
     "p95_request_seconds": 0.19794775801710784,
     "input_tokens": 20420,
     "output_tokens": 100,
     "cost_usd": 0.002042,
     "cost_per_million_questions": 20.42,
     "served_models": [
      "decision-models/baseten-mammoth-27b-latest"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6994407568940083,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:24:56+00:00",
     "ended": "2026-11-05T16:25:17+00:00",
     "questions": 100,
     "correct": 37,
     "accuracy": 0.37,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 3,
     "median_request_seconds": 0.18425167596433312,
     "p95_request_seconds": 0.21254337509162724,
     "input_tokens": 20420,
     "output_tokens": 100,
     "cost_usd": 0.002042,
     "cost_per_million_questions": 20.42,
     "served_models": [
      "decision-models/baseten-mammoth-27b-latest"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6994407568940083,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.37,
    "min": 0.37,
    "max": 0.37
   },
   "accuracy_ci95": {
    "low": 0.28,
    "high": 0.47
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.18436282750917599,
    "min": 0.18387077003717422,
    "max": 0.18496603652602062
   },
   "p95_request_seconds": {
    "mean": 0.2549800120371704,
    "min": 0.19794775801710784,
    "max": 0.3544489030027762
   },
   "cost_per_million_questions": {
    "mean": 20.42,
    "min": 20.42,
    "max": 20.42
   },
   "host": "Baseten",
   "run": 374,
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.6994454528762788,
    "min": 0.6994407568940083,
    "max": 0.6994548448408199
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.23,
     0.72,
     0.02,
     0.03
    ],
    "answers": 300,
    "yes_share": 0.23333333333333334,
    "first_share_other": 0.225,
    "baseline": 0.33924183006535946,
    "above": 0.030758169934640534,
    "p": 0.23000030245385794,
    "first_in_10": 2,
    "confidence": {
     "answers": 300,
     "mean_right": 0.7008854667931697,
     "mean_wrong": 0.6985997304171524,
     "sure": 0,
     "sure_wrong": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     0,
     0,
     0
    ],
    "hard_right_all_runs": 0
   }
  },
  {
   "provider": "baselabs",
   "model": "baselabs/elephant-1",
   "perplexity_mode": null,
   "request_model": "baselabs/elephant-1",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "Baseten's list price for the same model",
    "source": "https://www.baseten.co/library/elephant-1/",
    "date": "2026-11-01",
    "input_per_million": 0.035,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T20:31:23+00:00",
     "ended": "2026-11-05T20:31:28+00:00",
     "questions": 100,
     "correct": 33,
     "accuracy": 0.33,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.055772138526663184,
     "p95_request_seconds": 0.06140920508187264,
     "input_tokens": 20433,
     "output_tokens": 100,
     "cost_usd": 0.0007151550000000001,
     "cost_per_million_questions": 7.151550000000001,
     "served_models": [
      "baselabs/elephant-1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.68189809,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T20:31:28+00:00",
     "ended": "2026-11-05T20:31:34+00:00",
     "questions": 100,
     "correct": 33,
     "accuracy": 0.33,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.05640739051159471,
     "p95_request_seconds": 0.060983759933151305,
     "input_tokens": 20433,
     "output_tokens": 100,
     "cost_usd": 0.0007151550000000001,
     "cost_per_million_questions": 7.151550000000001,
     "served_models": [
      "baselabs/elephant-1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.68189809,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T20:31:34+00:00",
     "ended": "2026-11-05T20:31:40+00:00",
     "questions": 100,
     "correct": 33,
     "accuracy": 0.33,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.0525261455331929,
     "p95_request_seconds": 0.06694415002129972,
     "input_tokens": 20433,
     "output_tokens": 100,
     "cost_usd": 0.0007151550000000001,
     "cost_per_million_questions": 7.151550000000001,
     "served_models": [
      "baselabs/elephant-1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.68189809,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.33,
    "min": 0.33,
    "max": 0.33
   },
   "accuracy_ci95": {
    "low": 0.24,
    "high": 0.42
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.054901891523816936,
    "min": 0.0525261455331929,
    "max": 0.05640739051159471
   },
   "p95_request_seconds": {
    "mean": 0.06311237167877455,
    "min": 0.060983759933151305,
    "max": 0.06694415002129972
   },
   "cost_per_million_questions": {
    "mean": 7.151550000000001,
    "min": 7.151550000000001,
    "max": 7.151550000000001
   },
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.68189809,
    "min": 0.68189809,
    "max": 0.68189809
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "list-price estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.17,
     0.76,
     0.04,
     0.03
    ],
    "answers": 300,
    "yes_share": 0.13333333333333333,
    "first_share_other": 0.225,
    "baseline": 0.2984575163398693,
    "above": 0.03154248366013074,
    "p": 0.1944158438617486,
    "first_in_10": 2,
    "confidence": {
     "answers": 300,
     "mean_right": 0.6836922121212121,
     "mean_wrong": 0.6810144179104478,
     "sure": 0,
     "sure_wrong": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     0,
     0,
     0
    ],
    "hard_right_all_runs": 0,
    "weights": {
     "repo": "baselabs/elephant-1",
     "revision": "b3f8a93",
     "committed": "2026-10-22",
     "days_before": 11
    }
   }
  },
  {
   "provider": "baselabs",
   "model": "baselabs/mammoth-1",
   "perplexity_mode": null,
   "request_model": "baselabs/mammoth-1",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "Baseten's list price for the same model",
    "source": "https://www.baseten.co/library/mammoth-1/",
    "date": "2026-11-01",
    "input_per_million": 0.1,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T20:30:05+00:00",
     "ended": "2026-11-05T20:30:18+00:00",
     "questions": 100,
     "correct": 39,
     "accuracy": 0.39,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.1284455395070836,
     "p95_request_seconds": 0.13529617397580296,
     "input_tokens": 20433,
     "output_tokens": 100,
     "cost_usd": 0.0020433,
     "cost_per_million_questions": 20.433,
     "served_models": [
      "baselabs/mammoth-1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.69924914,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T20:30:18+00:00",
     "ended": "2026-11-05T20:30:31+00:00",
     "questions": 100,
     "correct": 39,
     "accuracy": 0.39,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.12809407198801637,
     "p95_request_seconds": 0.13415572594385594,
     "input_tokens": 20433,
     "output_tokens": 100,
     "cost_usd": 0.0020433,
     "cost_per_million_questions": 20.433,
     "served_models": [
      "baselabs/mammoth-1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.69924914,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T20:30:31+00:00",
     "ended": "2026-11-05T20:30:44+00:00",
     "questions": 100,
     "correct": 39,
     "accuracy": 0.39,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.127208921010606,
     "p95_request_seconds": 0.1341511399950832,
     "input_tokens": 20433,
     "output_tokens": 100,
     "cost_usd": 0.0020433,
     "cost_per_million_questions": 20.433,
     "served_models": [
      "baselabs/mammoth-1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.69924914,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.39,
    "min": 0.39,
    "max": 0.39
   },
   "accuracy_ci95": {
    "low": 0.3,
    "high": 0.49
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.12791617750190198,
    "min": 0.127208921010606,
    "max": 0.1284455395070836
   },
   "p95_request_seconds": {
    "mean": 0.13453434663824737,
    "min": 0.1341511399950832,
    "max": 0.13529617397580296
   },
   "cost_per_million_questions": {
    "mean": 20.433,
    "min": 20.433,
    "max": 20.433
   },
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.69924914,
    "min": 0.69924914,
    "max": 0.69924914
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "list-price estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.24,
     0.71,
     0.02,
     0.03
    ],
    "answers": 300,
    "yes_share": 0.25,
    "first_share_other": 0.225,
    "baseline": 0.3549084967320261,
    "above": 0.03509150326797389,
    "p": 0.20233956975622092,
    "first_in_10": 2,
    "confidence": {
     "answers": 300,
     "mean_right": 0.6831373076923077,
     "mean_wrong": 0.7095501475409836,
     "sure": 0,
     "sure_wrong": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     0,
     0,
     0
    ],
    "hard_right_all_runs": 0,
    "weights": {
     "repo": "baselabs/mammoth-1",
     "revision": "61cec44",
     "committed": "2026-10-22",
     "days_before": 11
    }
   }
  },
  {
   "provider": "pplx-decider",
   "model": "pplx-decider-v1.2",
   "perplexity_mode": null,
   "request_model": "pplx-decider-v1.2",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "Perplexity's list price for the same model",
    "source": "https://docs.perplexity.ai/docs/getting-started/pricing",
    "date": "2026-10-18",
    "input_per_million": 0.02,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T20:28:02+00:00",
     "ended": "2026-11-05T20:28:11+00:00",
     "questions": 100,
     "correct": 57,
     "accuracy": 0.57,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.0744166465010494,
     "p95_request_seconds": 0.09870172303635627,
     "input_tokens": 19838,
     "output_tokens": 0,
     "cost_usd": 0.00040276,
     "cost_per_million_questions": 4.0276000000000005,
     "served_models": [
      "pplx-decider-v1.2"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6545199242428439,
      "sure": 1,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T20:28:11+00:00",
     "ended": "2026-11-05T20:28:19+00:00",
     "questions": 100,
     "correct": 57,
     "accuracy": 0.57,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.0736691135680303,
     "p95_request_seconds": 0.09842301101889461,
     "input_tokens": 19838,
     "output_tokens": 0,
     "cost_usd": 0.00040276,
     "cost_per_million_questions": 4.0276000000000005,
     "served_models": [
      "pplx-decider-v1.2"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6545199242428439,
      "sure": 1,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T20:28:19+00:00",
     "ended": "2026-11-05T20:28:26+00:00",
     "questions": 100,
     "correct": 57,
     "accuracy": 0.57,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.07364938355749473,
     "p95_request_seconds": 0.0982112199999392,
     "input_tokens": 19838,
     "output_tokens": 0,
     "cost_usd": 0.00040276,
     "cost_per_million_questions": 4.0276000000000005,
     "served_models": [
      "pplx-decider-v1.2"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6545199242428439,
      "sure": 1,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.57,
    "min": 0.57,
    "max": 0.57
   },
   "accuracy_ci95": {
    "low": 0.47,
    "high": 0.66
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.07391171454219148,
    "min": 0.07364938355749473,
    "max": 0.0744166465010494
   },
   "p95_request_seconds": {
    "mean": 0.09844531801839669,
    "min": 0.0982112199999392,
    "max": 0.09870172303635627
   },
   "cost_per_million_questions": {
    "mean": 4.0276000000000005,
    "min": 4.0276000000000005,
    "max": 4.0276000000000005
   },
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.6545199242428439,
    "min": 0.6545199242428439,
    "max": 0.6545199242428439
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 1,
    "min": 1,
    "max": 1
   },
   "cost_kind": "list-price estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.52,
     0.37,
     0.08,
     0.03
    ],
    "answers": 300,
    "yes_share": 0.65,
    "first_share_other": 0.325,
    "baseline": 0.4999281045751634,
    "above": 0.07007189542483655,
    "p": 0.0663399578443511,
    "first_in_10": 5,
    "confidence": {
     "answers": 300,
     "mean_right": 0.6823476737240217,
     "mean_wrong": 0.6176319772561664,
     "sure": 3,
     "sure_wrong": 0
    },
    "reversed": {
     "run_1_accuracy": 0.57,
     "accuracy": 0.55,
     "change": -0.02,
     "same_choice_as_run_1": 0.93,
     "runs_1_2_agree": 1.0,
     "errors": 0,
     "missing": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     1,
     1,
     1
    ],
    "hard_right_all_runs": 1,
    "weights": {
     "repo": "perplexity-ai/pplx-decider-v1.2",
     "revision": "225c913",
     "committed": "2026-10-16",
     "days_before": 17
    }
   }
  },
  {
   "provider": "pplx-decider",
   "model": "pplx-decider-flash-v1.2",
   "perplexity_mode": null,
   "request_model": "pplx-decider-flash-v1.2",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "Perplexity's list price for the same model",
    "source": "https://docs.perplexity.ai/docs/getting-started/pricing",
    "date": "2026-10-18",
    "input_per_million": 0.01,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T20:25:12+00:00",
     "ended": "2026-11-05T20:25:15+00:00",
     "questions": 100,
     "correct": 59,
     "accuracy": 0.59,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.027386887522879988,
     "p95_request_seconds": 0.030885283020325005,
     "input_tokens": 19838,
     "output_tokens": 0,
     "cost_usd": 0.00020138,
     "cost_per_million_questions": 2.0138000000000003,
     "served_models": [
      "pplx-decider-flash-v1.2"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6677402274970063,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T20:25:15+00:00",
     "ended": "2026-11-05T20:25:18+00:00",
     "questions": 100,
     "correct": 59,
     "accuracy": 0.59,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.02720481646247208,
     "p95_request_seconds": 0.02940976805984974,
     "input_tokens": 19838,
     "output_tokens": 0,
     "cost_usd": 0.00020138,
     "cost_per_million_questions": 2.0138000000000003,
     "served_models": [
      "pplx-decider-flash-v1.2"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6677402274970063,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T20:25:18+00:00",
     "ended": "2026-11-05T20:25:21+00:00",
     "questions": 100,
     "correct": 59,
     "accuracy": 0.59,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.027039420034270734,
     "p95_request_seconds": 0.029818207025527954,
     "input_tokens": 19838,
     "output_tokens": 0,
     "cost_usd": 0.00020138,
     "cost_per_million_questions": 2.0138000000000003,
     "served_models": [
      "pplx-decider-flash-v1.2"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6677402274970063,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.59,
    "min": 0.59,
    "max": 0.59
   },
   "accuracy_ci95": {
    "low": 0.5,
    "high": 0.69
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.0272103746732076,
    "min": 0.027039420034270734,
    "max": 0.027386887522879988
   },
   "p95_request_seconds": {
    "mean": 0.0300377527019009,
    "min": 0.02940976805984974,
    "max": 0.030885283020325005
   },
   "cost_per_million_questions": {
    "mean": 2.0138000000000003,
    "min": 2.0138000000000003,
    "max": 2.0138000000000003
   },
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.6677402274970063,
    "min": 0.6677402274970063,
    "max": 0.6677402274970063
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "list-price estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.66,
     0.26,
     0.05,
     0.03
    ],
    "answers": 300,
    "yes_share": 0.7833333333333333,
    "first_share_other": 0.475,
    "baseline": 0.5815163398692811,
    "above": 0.008483660130718884,
    "p": 0.4212772401814321,
    "first_in_10": 7,
    "confidence": {
     "answers": 300,
     "mean_right": 0.6994089971292216,
     "mean_wrong": 0.6221680955872332,
     "sure": 0,
     "sure_wrong": 0
    },
    "reversed": {
     "run_1_accuracy": 0.59,
     "accuracy": 0.62,
     "change": 0.03,
     "same_choice_as_run_1": 0.95,
     "runs_1_2_agree": 1.0,
     "errors": 0,
     "missing": 0
    },
    "p_holm": 1.0,
    "explained": true,
    "hard_right": [
     2,
     2,
     2
    ],
    "hard_right_all_runs": 2,
    "weights": {
     "repo": "perplexity-ai/pplx-decider-flash-v1.2",
     "revision": "6d771ad",
     "committed": "2026-10-16",
     "days_before": 17
    }
   }
  },
  {
   "provider": "glide",
   "model": "glide-v1.2",
   "perplexity_mode": null,
   "request_model": "glide-v1.2",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "Fastino's list price for the same model",
    "source": "https://docs.fastino.ai/pricing",
    "date": "2026-10-22",
    "input_per_million": 0.04,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T20:28:57+00:00",
     "ended": "2026-11-05T20:29:05+00:00",
     "questions": 100,
     "correct": 53,
     "accuracy": 0.53,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.06414138502441347,
     "p95_request_seconds": 0.0764834430301562,
     "input_tokens": 17238,
     "output_tokens": 0,
     "cost_usd": 0.00068952,
     "cost_per_million_questions": 6.8952,
     "served_models": [
      "glide-v1.2"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6763361985957483,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T20:29:05+00:00",
     "ended": "2026-11-05T20:29:11+00:00",
     "questions": 100,
     "correct": 53,
     "accuracy": 0.53,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.06368989148177207,
     "p95_request_seconds": 0.06748599803540856,
     "input_tokens": 17238,
     "output_tokens": 0,
     "cost_usd": 0.00068952,
     "cost_per_million_questions": 6.8952,
     "served_models": [
      "glide-v1.2"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6763361985957483,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T20:29:11+00:00",
     "ended": "2026-11-05T20:29:17+00:00",
     "questions": 100,
     "correct": 53,
     "accuracy": 0.53,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.06347203551558778,
     "p95_request_seconds": 0.06836205907166004,
     "input_tokens": 17238,
     "output_tokens": 0,
     "cost_usd": 0.00068952,
     "cost_per_million_questions": 6.8952,
     "served_models": [
      "glide-v1.2"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6763361985957483,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.53,
    "min": 0.53,
    "max": 0.53
   },
   "accuracy_ci95": {
    "low": 0.43,
    "high": 0.63
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.06376777067392443,
    "min": 0.06347203551558778,
    "max": 0.06414138502441347
   },
   "p95_request_seconds": {
    "mean": 0.07077716671240826,
    "min": 0.06748599803540856,
    "max": 0.0764834430301562
   },
   "cost_per_million_questions": {
    "mean": 6.8952,
    "min": 6.8952,
    "max": 6.8952
   },
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.6763361985957483,
    "min": 0.6763361985957483,
    "max": 0.6763361985957483
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "list-price estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.47,
     0.45,
     0.05,
     0.03
    ],
    "answers": 300,
    "yes_share": 0.5666666666666667,
    "first_share_other": 0.325,
    "baseline": 0.4691437908496732,
    "above": 0.0608562091503268,
    "p": 0.10127444925698104,
    "first_in_10": 5,
    "confidence": {
     "answers": 300,
     "mean_right": 0.6888612732869043,
     "mean_wrong": 0.6622121781993383,
     "sure": 0,
     "sure_wrong": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     1,
     1,
     1
    ],
    "hard_right_all_runs": 1,
    "weights": {
     "repo": "fastino/GLiDE",
     "revision": "a7e4589",
     "committed": "2026-10-22",
     "days_before": 11
    }
   }
  },
  {
   "provider": "vllm-sr",
   "model": "vllm-sr/d3",
   "perplexity_mode": null,
   "request_model": "vllm-sr/d3",
   "group": "decision",
   "web": false,
   "price": null,
   "runs": [
    {
     "started": "2026-11-05T20:27:46+00:00",
     "ended": "2026-11-05T20:27:52+00:00",
     "questions": 100,
     "correct": 58,
     "accuracy": 0.58,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.06280192051781341,
     "p95_request_seconds": 0.0763826509937644,
     "input_tokens": 20638,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "vllm-sr/d3"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6604659003471288,
      "sure": 2,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T20:27:52+00:00",
     "ended": "2026-11-05T20:27:58+00:00",
     "questions": 100,
     "correct": 58,
     "accuracy": 0.58,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.06308425852330402,
     "p95_request_seconds": 0.07674432802014053,
     "input_tokens": 20638,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "vllm-sr/d3"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6604659003471288,
      "sure": 2,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T20:27:58+00:00",
     "ended": "2026-11-05T20:28:04+00:00",
     "questions": 100,
     "correct": 58,
     "accuracy": 0.58,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.06285491603193805,
     "p95_request_seconds": 0.07670124399010092,
     "input_tokens": 20638,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "vllm-sr/d3"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6604659003471288,
      "sure": 2,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.58,
    "min": 0.58,
    "max": 0.58
   },
   "accuracy_ci95": {
    "low": 0.49,
    "high": 0.68
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.06291369835768516,
    "min": 0.06280192051781341,
    "max": 0.06308425852330402
   },
   "p95_request_seconds": {
    "mean": 0.07660940766800195,
    "min": 0.0763826509937644,
    "max": 0.07674432802014053
   },
   "cost_per_million_questions": null,
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.6604659003471288,
    "min": 0.6604659003471288,
    "max": 0.6604659003471288
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 2,
    "min": 2,
    "max": 2
   },
   "cost_kind": null,
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.55,
     0.37,
     0.05,
     0.03
    ],
    "answers": 300,
    "yes_share": 0.6166666666666667,
    "first_share_other": 0.45,
    "baseline": 0.5149477124183006,
    "above": 0.06505228758169934,
    "p": 0.0849749713940291,
    "first_in_10": 6,
    "confidence": {
     "answers": 300,
     "mean_right": 0.6802326606298315,
     "mean_wrong": 0.6331689456710157,
     "sure": 6,
     "sure_wrong": 0
    },
    "reversed": {
     "run_1_accuracy": 0.58,
     "accuracy": 0.57,
     "change": -0.01,
     "same_choice_as_run_1": 0.95,
     "runs_1_2_agree": 1.0,
     "errors": 0,
     "missing": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     0,
     0,
     0
    ],
    "hard_right_all_runs": 0,
    "weights": {
     "repo": "vllm-sr/d3",
     "revision": "20f37b9",
     "committed": "2026-10-22",
     "days_before": 11
    }
   }
  },
  {
   "provider": "d3-server",
   "model": "vllm-sr/d3-ultra",
   "perplexity_mode": null,
   "request_model": "vllm-sr/d3-ultra",
   "group": "decision",
   "web": false,
   "price": null,
   "runs": [
    {
     "started": "2026-11-05T20:30:39+00:00",
     "ended": "2026-11-05T20:30:52+00:00",
     "questions": 100,
     "correct": 55,
     "accuracy": 0.55,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.12481923052109778,
     "p95_request_seconds": 0.17127542896196246,
     "input_tokens": 30125,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "d3-ultra"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.788696731856307,
      "sure": 2,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T20:30:52+00:00",
     "ended": "2026-11-05T20:31:05+00:00",
     "questions": 100,
     "correct": 55,
     "accuracy": 0.55,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.12525110505521297,
     "p95_request_seconds": 0.17096274497453123,
     "input_tokens": 30125,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "d3-ultra"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.788696731856307,
      "sure": 2,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T20:31:06+00:00",
     "ended": "2026-11-05T20:31:18+00:00",
     "questions": 100,
     "correct": 55,
     "accuracy": 0.55,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.1246361979865469,
     "p95_request_seconds": 0.17062305996660143,
     "input_tokens": 30125,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "d3-ultra"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.788696731856307,
      "sure": 2,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.55,
    "min": 0.55,
    "max": 0.55
   },
   "accuracy_ci95": {
    "low": 0.45,
    "high": 0.65
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.12490217785428588,
    "min": 0.1246361979865469,
    "max": 0.12525110505521297
   },
   "p95_request_seconds": {
    "mean": 0.17095374463436505,
    "min": 0.17062305996660143,
    "max": 0.17127542896196246
   },
   "cost_per_million_questions": null,
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.788696731856307,
    "min": 0.788696731856307,
    "max": 0.788696731856307
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 2,
    "min": 2,
    "max": 2
   },
   "cost_kind": null,
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.48,
     0.44,
     0.06,
     0.02
    ],
    "answers": 300,
    "yes_share": 0.55,
    "first_share_other": 0.375,
    "baseline": 0.479718954248366,
    "above": 0.07028104575163402,
    "p": 0.07521939803292399,
    "first_in_10": 5,
    "confidence": {
     "answers": 300,
     "mean_right": 0.8148138123190375,
     "mean_wrong": 0.7567758557351917,
     "sure": 6,
     "sure_wrong": 0
    },
    "reversed": {
     "run_1_accuracy": 0.55,
     "accuracy": 0.55,
     "change": 0.0,
     "same_choice_as_run_1": 0.93,
     "runs_1_2_agree": 1.0,
     "errors": 0,
     "missing": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     1,
     1,
     1
    ],
    "hard_right_all_runs": 1,
    "weights": {
     "repo": "vllm-sr/d3-ultra",
     "revision": "52ba9e7",
     "committed": "2026-10-26",
     "days_before": 7
    }
   }
  },
  {
   "provider": "drex-max",
   "model": "drex-max-v1",
   "perplexity_mode": null,
   "request_model": "drex-max-v1",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "Nace.AI's list price for the same model",
    "source": "https://nace.ai/drex",
    "date": "2026-10-30",
    "input_per_million": 0.06,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T20:32:48+00:00",
     "ended": "2026-11-05T20:32:54+00:00",
     "questions": 100,
     "correct": 42,
     "accuracy": 0.42,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.06390973302768543,
     "p95_request_seconds": 0.0783776449970901,
     "input_tokens": 20138,
     "output_tokens": 0,
     "cost_usd": 0.00120828,
     "cost_per_million_questions": 12.082799999999999,
     "served_models": [
      "drex-max-v1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.8177372988891587,
      "sure": 12,
      "sure_wrong": 7
     }
    },
    {
     "started": "2026-11-05T20:32:54+00:00",
     "ended": "2026-11-05T20:33:01+00:00",
     "questions": 100,
     "correct": 42,
     "accuracy": 0.42,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.06416410848032683,
     "p95_request_seconds": 0.0783548200270161,
     "input_tokens": 20138,
     "output_tokens": 0,
     "cost_usd": 0.00120828,
     "cost_per_million_questions": 12.082799999999999,
     "served_models": [
      "drex-max-v1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.8177372988891587,
      "sure": 12,
      "sure_wrong": 7
     }
    },
    {
     "started": "2026-11-05T20:33:01+00:00",
     "ended": "2026-11-05T20:33:07+00:00",
     "questions": 100,
     "correct": 42,
     "accuracy": 0.42,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.06400984950596467,
     "p95_request_seconds": 0.078073943965137,
     "input_tokens": 20138,
     "output_tokens": 0,
     "cost_usd": 0.00120828,
     "cost_per_million_questions": 12.082799999999999,
     "served_models": [
      "drex-max-v1"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.8177372988891587,
      "sure": 12,
      "sure_wrong": 7
     }
    }
   ],
   "accuracy": {
    "mean": 0.42,
    "min": 0.42,
    "max": 0.42
   },
   "accuracy_ci95": {
    "low": 0.32,
    "high": 0.52
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.06402789700465898,
    "min": 0.06390973302768543,
    "max": 0.06416410848032683
   },
   "p95_request_seconds": {
    "mean": 0.07826880299641441,
    "min": 0.078073943965137,
    "max": 0.0783776449970901
   },
   "cost_per_million_questions": {
    "mean": 12.082799999999999,
    "min": 12.082799999999999,
    "max": 12.082799999999999
   },
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.8177372988891587,
    "min": 0.8177372988891587,
    "max": 0.8177372988891587
   },
   "sure_wrong": {
    "mean": 7,
    "min": 7,
    "max": 7
   },
   "sure": {
    "mean": 12,
    "min": 12,
    "max": 12
   },
   "cost_kind": "list-price estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.34,
     0.59,
     0.04,
     0.03
    ],
    "answers": 300,
    "yes_share": 0.38333333333333336,
    "first_share_other": 0.275,
    "baseline": 0.40758169934640526,
    "above": 0.012418300653594727,
    "p": 0.3944028112369925,
    "first_in_10": 3,
    "confidence": {
     "answers": 300,
     "mean_right": 0.8309167377522277,
     "mean_wrong": 0.8081935672986603,
     "sure": 36,
     "sure_wrong": 21
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     0,
     0,
     0
    ],
    "hard_right_all_runs": 0,
    "weights": {
     "repo": "nace-ai/drex-max-v1",
     "revision": "cac65e1",
     "committed": "2026-10-30",
     "days_before": 3
    }
   }
  },
  {
   "provider": "clef",
   "model": "clef",
   "perplexity_mode": null,
   "request_model": "clef",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "Cloudflare's list price for the same model",
    "source": "https://developers.cloudflare.com/workers-ai/models/clef/",
    "date": "2026-10-18",
    "input_per_million": 0.24,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T20:23:00+00:00",
     "ended": "2026-11-05T20:23:13+00:00",
     "questions": 100,
     "correct": 52,
     "accuracy": 0.52,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.12826486048288643,
     "p95_request_seconds": 0.15441878593992442,
     "input_tokens": 24334,
     "output_tokens": 0,
     "cost_usd": 0.00584016,
     "cost_per_million_questions": 58.401599999999995,
     "served_models": [
      "clef"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.680091,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T20:23:13+00:00",
     "ended": "2026-11-05T20:23:26+00:00",
     "questions": 100,
     "correct": 52,
     "accuracy": 0.52,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.12628116947598755,
     "p95_request_seconds": 0.1533330649835989,
     "input_tokens": 24334,
     "output_tokens": 0,
     "cost_usd": 0.00584016,
     "cost_per_million_questions": 58.401599999999995,
     "served_models": [
      "clef"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.680091,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T20:23:26+00:00",
     "ended": "2026-11-05T20:23:39+00:00",
     "questions": 100,
     "correct": 52,
     "accuracy": 0.52,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.12346055649686605,
     "p95_request_seconds": 0.15348530700430274,
     "input_tokens": 24334,
     "output_tokens": 0,
     "cost_usd": 0.00584016,
     "cost_per_million_questions": 58.401599999999995,
     "served_models": [
      "clef"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.680091,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.52,
    "min": 0.52,
    "max": 0.52
   },
   "accuracy_ci95": {
    "low": 0.42,
    "high": 0.62
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.1260021954852467,
    "min": 0.12346055649686605,
    "max": 0.12826486048288643
   },
   "p95_request_seconds": {
    "mean": 0.15374571930927536,
    "min": 0.1533330649835989,
    "max": 0.15441878593992442
   },
   "cost_per_million_questions": {
    "mean": 58.401599999999995,
    "min": 58.401599999999995,
    "max": 58.401599999999995
   },
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.680091,
    "min": 0.680091,
    "max": 0.680091
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "list-price estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.4,
     0.52,
     0.05,
     0.03
    ],
    "answers": 300,
    "yes_share": 0.38333333333333336,
    "first_share_other": 0.425,
    "baseline": 0.4134901960784314,
    "above": 0.10650980392156861,
    "p": 0.012999750414502742,
    "first_in_10": 4,
    "confidence": {
     "answers": 300,
     "mean_right": 0.6647846153846154,
     "mean_wrong": 0.6966729166666666,
     "sure": 0,
     "sure_wrong": 0
    },
    "reversed": {
     "run_1_accuracy": 0.52,
     "accuracy": 0.52,
     "change": 0.0,
     "same_choice_as_run_1": 1.0,
     "runs_1_2_agree": 1.0,
     "errors": 0,
     "missing": 0
    },
    "p_holm": 0.48099076533660146,
    "explained": false,
    "hard_right": [
     3,
     3,
     3
    ],
    "hard_right_all_runs": 3,
    "weights": {
     "repo": "Cloudflare/clef",
     "revision": "2f3de3d",
     "committed": "2026-10-01",
     "days_before": 32
    }
   }
  },
  {
   "provider": "clef",
   "model": "clef-flash",
   "perplexity_mode": null,
   "request_model": "clef-flash",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "Cloudflare's list price for the same model",
    "source": "https://developers.cloudflare.com/workers-ai/models/clef-flash/",
    "date": "2026-10-18",
    "input_per_million": 0.038,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T16:16:51+00:00",
     "ended": "2026-11-05T16:16:57+00:00",
     "questions": 100,
     "correct": 48,
     "accuracy": 0.48,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.06602641200879589,
     "p95_request_seconds": 0.06978178594727069,
     "input_tokens": 24334,
     "output_tokens": 0,
     "cost_usd": 0.000924692,
     "cost_per_million_questions": 9.24692,
     "served_models": [
      "clef-flash"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.69787,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:16:57+00:00",
     "ended": "2026-11-05T16:17:05+00:00",
     "questions": 100,
     "correct": 48,
     "accuracy": 0.48,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.06702545995358378,
     "p95_request_seconds": 0.12130392901599407,
     "input_tokens": 24334,
     "output_tokens": 0,
     "cost_usd": 0.000924692,
     "cost_per_million_questions": 9.24692,
     "served_models": [
      "clef-flash"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.69787,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:17:05+00:00",
     "ended": "2026-11-05T16:17:17+00:00",
     "questions": 100,
     "correct": 48,
     "accuracy": 0.48,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.12132179143372923,
     "p95_request_seconds": 0.13657160894945264,
     "input_tokens": 24334,
     "output_tokens": 0,
     "cost_usd": 0.000924692,
     "cost_per_million_questions": 9.24692,
     "served_models": [
      "clef-flash"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.69787,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.48,
    "min": 0.48,
    "max": 0.48
   },
   "accuracy_ci95": {
    "low": 0.38,
    "high": 0.58
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.0847912211320363,
    "min": 0.06602641200879589,
    "max": 0.12132179143372923
   },
   "p95_request_seconds": {
    "mean": 0.1092191079709058,
    "min": 0.06978178594727069,
    "max": 0.13657160894945264
   },
   "cost_per_million_questions": {
    "mean": 9.24692,
    "min": 9.24692,
    "max": 9.24692
   },
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.69787,
    "min": 0.69787,
    "max": 0.69787
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "list-price estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.47,
     0.45,
     0.05,
     0.03
    ],
    "answers": 300,
    "yes_share": 0.5166666666666667,
    "first_share_other": 0.4,
    "baseline": 0.45260784313725494,
    "above": 0.027392156862745043,
    "p": 0.2867576011825844,
    "first_in_10": 5,
    "confidence": {
     "answers": 300,
     "mean_right": 0.7187625,
     "mean_wrong": 0.6785846153846153,
     "sure": 0,
     "sure_wrong": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     0,
     0,
     0
    ],
    "hard_right_all_runs": 0,
    "weights": {
     "repo": "Cloudflare/clef-flash",
     "revision": "fde727a",
     "committed": "2026-10-07",
     "note": "weights 2026-09-30",
     "days_before": 26
    }
   }
  },
  {
   "provider": "cygnet",
   "model": "cygnet",
   "perplexity_mode": null,
   "request_model": "cygnet",
   "group": "decision",
   "web": false,
   "price": {
    "basis": "SiliconFlow's list price for the same weights (Gemma 4 12B IT; precision not stated, ours BF16)",
    "source": "https://www.siliconflow.com/models/gemma-4-12b-it",
    "date": "2026-11-05",
    "input_per_million": 0.1,
    "per_request": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T19:28:06+00:00",
     "ended": "2026-11-05T19:28:09+00:00",
     "questions": 100,
     "correct": 32,
     "accuracy": 0.32,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.03270593751221895,
     "p95_request_seconds": 0.03558983898255974,
     "input_tokens": 22522,
     "output_tokens": 100,
     "cost_usd": 0.0022822000000000003,
     "cost_per_million_questions": 22.822000000000003,
     "served_models": [
      "cygnet"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.804577812678529,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T19:28:10+00:00",
     "ended": "2026-11-05T19:28:12+00:00",
     "questions": 100,
     "correct": 32,
     "accuracy": 0.32,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.02448406192706898,
     "p95_request_seconds": 0.025801007985137403,
     "input_tokens": 22522,
     "output_tokens": 100,
     "cost_usd": 0.0022822000000000003,
     "cost_per_million_questions": 22.822000000000003,
     "served_models": [
      "cygnet"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.8044741910413394,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T19:28:12+00:00",
     "ended": "2026-11-05T19:28:15+00:00",
     "questions": 100,
     "correct": 32,
     "accuracy": 0.32,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.024432203033939004,
     "p95_request_seconds": 0.026117929024621844,
     "input_tokens": 22522,
     "output_tokens": 100,
     "cost_usd": 0.0022822000000000003,
     "cost_per_million_questions": 22.822000000000003,
     "served_models": [
      "cygnet"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.8044741910413394,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.32,
    "min": 0.32,
    "max": 0.32
   },
   "accuracy_ci95": {
    "low": 0.23,
    "high": 0.41
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.027207400824408978,
    "min": 0.024432203033939004,
    "max": 0.03270593751221895
   },
   "p95_request_seconds": {
    "mean": 0.029169591997439664,
    "min": 0.025801007985137403,
    "max": 0.03558983898255974
   },
   "cost_per_million_questions": {
    "mean": 22.822000000000003,
    "min": 22.822000000000003,
    "max": 22.822000000000003
   },
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.8045087315870693,
    "min": 0.8044741910413394,
    "max": 0.804577812678529
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "list-price estimate",
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.15,
     0.72,
     0.04,
     0.09
    ],
    "answers": 300,
    "yes_share": 0.18333333333333332,
    "first_share_other": 0.1,
    "baseline": 0.2923398692810457,
    "above": 0.02766013071895429,
    "p": 0.22630556033196825,
    "first_in_10": 2,
    "confidence": {
     "answers": 300,
     "mean_right": 0.8054262359361548,
     "mean_wrong": 0.8040769648345585,
     "sure": 0,
     "sure_wrong": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     1,
     1,
     1
    ],
    "hard_right_all_runs": 1,
    "weights": {
     "repo": "google/gemma-4-12B-it",
     "revision": "707f0a3",
     "committed": "2026-07-20",
     "note": "Cygnet runs it unchanged, with its own sampling recipe",
     "days_before": 105
    }
   }
  },
  {
   "provider": "open-jev",
   "model": "open-jev-27b-v1.1",
   "perplexity_mode": null,
   "request_model": "open-jev-27b-v1.1",
   "group": "decision",
   "web": false,
   "price": null,
   "runs": [
    {
     "started": "2026-11-05T20:02:04+00:00",
     "ended": "2026-11-05T20:02:21+00:00",
     "questions": 100,
     "correct": 49,
     "accuracy": 0.49,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.1466473464970477,
     "p95_request_seconds": 0.2713994199875742,
     "input_tokens": 34133,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "Qwen/Qwen3.8-27B"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.5324695239499188,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T20:02:21+00:00",
     "ended": "2026-11-05T20:02:37+00:00",
     "questions": 100,
     "correct": 49,
     "accuracy": 0.49,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.14577156503219157,
     "p95_request_seconds": 0.27216335700359195,
     "input_tokens": 34133,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "Qwen/Qwen3.8-27B"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.5324695239499188,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T20:02:38+00:00",
     "ended": "2026-11-05T20:02:54+00:00",
     "questions": 100,
     "correct": 49,
     "accuracy": 0.49,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.14482618344482034,
     "p95_request_seconds": 0.27166756498627365,
     "input_tokens": 34133,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "Qwen/Qwen3.8-27B"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.5324695239499188,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.49,
    "min": 0.49,
    "max": 0.49
   },
   "accuracy_ci95": {
    "low": 0.39,
    "high": 0.59
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.1457483649913532,
    "min": 0.14482618344482034,
    "max": 0.1466473464970477
   },
   "p95_request_seconds": {
    "mean": 0.2717434473258133,
    "min": 0.2713994199875742,
    "max": 0.27216335700359195
   },
   "cost_per_million_questions": null,
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.5324695239499188,
    "min": 0.5324695239499188,
    "max": 0.5324695239499188
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": null,
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.53,
     0.37,
     0.08,
     0.02
    ],
    "answers": 300,
    "yes_share": 0.5833333333333334,
    "first_share_other": 0.45,
    "baseline": 0.49016339869281045,
    "above": -0.00016339869281045694,
    "p": 0.5013513642637362,
    "first_in_10": 5,
    "confidence": {
     "answers": 300,
     "mean_right": 0.5430061357313303,
     "mean_wrong": 0.5223461126305232,
     "sure": 0,
     "sure_wrong": 0
    },
    "p_holm": 1.0,
    "explained": true,
    "hard_right": [
     1,
     1,
     1
    ],
    "hard_right_all_runs": 1,
    "weights": {
     "repo": "ZefanCai/Open-Jev-27B-v1.1",
     "revision": "28cf730",
     "committed": "2026-09-23",
     "note": "on Qwen/Qwen3.8-27B at 1d4bf0f, 2026-08-14",
     "days_before": 40
    }
   }
  },
  {
   "provider": "winnow",
   "model": "winnow-12b",
   "perplexity_mode": null,
   "request_model": "winnow-12b",
   "group": "decision",
   "web": false,
   "price": null,
   "runs": [
    {
     "started": "2026-11-05T19:22:43+00:00",
     "ended": "2026-11-05T19:22:48+00:00",
     "questions": 100,
     "correct": 39,
     "accuracy": 0.39,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.047384657023940235,
     "p95_request_seconds": 0.05865891801659018,
     "input_tokens": 21750,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "winnow-12b"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7667033322145941,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T19:22:48+00:00",
     "ended": "2026-11-05T19:22:53+00:00",
     "questions": 100,
     "correct": 39,
     "accuracy": 0.39,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.04762945353286341,
     "p95_request_seconds": 0.058727662079036236,
     "input_tokens": 21750,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "winnow-12b"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7667033322145941,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T19:22:53+00:00",
     "ended": "2026-11-05T19:22:57+00:00",
     "questions": 100,
     "correct": 39,
     "accuracy": 0.39,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.04788949200883508,
     "p95_request_seconds": 0.058404942974448204,
     "input_tokens": 21750,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "winnow-12b"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7667033322145941,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.39,
    "min": 0.39,
    "max": 0.39
   },
   "accuracy_ci95": {
    "low": 0.29,
    "high": 0.49
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.04763453418854624,
    "min": 0.047384657023940235,
    "max": 0.04788949200883508
   },
   "p95_request_seconds": {
    "mean": 0.05859717435669154,
    "min": 0.058404942974448204,
    "max": 0.058727662079036236
   },
   "cost_per_million_questions": null,
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.7667033322145941,
    "min": 0.7667033322145941,
    "max": 0.7667033322145941
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": null,
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.28,
     0.61,
     0.03,
     0.08
    ],
    "answers": 300,
    "yes_share": 0.35,
    "first_share_other": 0.175,
    "baseline": 0.36679084967320263,
    "above": 0.02320915032679738,
    "p": 0.29944642171366376,
    "first_in_10": 3,
    "confidence": {
     "answers": 300,
     "mean_right": 0.7610219738479799,
     "mean_wrong": 0.7703356760883311,
     "sure": 0,
     "sure_wrong": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     1,
     1,
     1
    ],
    "hard_right_all_runs": 1,
    "weights": {
     "repo": "EldanRing/Winnow-12B",
     "revision": "b2b1421",
     "committed": "2026-09-21",
     "note": "Q8_0 GGUF; revision from its manifest, file hash not checked",
     "days_before": 42
    }
   }
  },
  {
   "provider": "metask",
   "model": "metask-jev-4b-policy-mix",
   "perplexity_mode": null,
   "request_model": "metask-jev-4b-policy-mix",
   "group": "decision",
   "web": false,
   "price": null,
   "runs": [
    {
     "started": "2026-11-05T19:17:40+00:00",
     "ended": "2026-11-05T19:17:48+00:00",
     "questions": 100,
     "correct": 60,
     "accuracy": 0.6,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.07358264405047521,
     "p95_request_seconds": 0.07851609797216952,
     "input_tokens": 28188,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "metask-jev-4b-policy-mix"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7406595665216446,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T19:17:48+00:00",
     "ended": "2026-11-05T19:17:55+00:00",
     "questions": 100,
     "correct": 60,
     "accuracy": 0.6,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.07402580551570281,
     "p95_request_seconds": 0.07952634803950787,
     "input_tokens": 28188,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "metask-jev-4b-policy-mix"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7406595665216446,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T19:17:55+00:00",
     "ended": "2026-11-05T19:18:03+00:00",
     "questions": 100,
     "correct": 60,
     "accuracy": 0.6,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.07346713898004964,
     "p95_request_seconds": 0.08013061305973679,
     "input_tokens": 28188,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "metask-jev-4b-policy-mix"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7406595665216446,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.6,
    "min": 0.6,
    "max": 0.6
   },
   "accuracy_ci95": {
    "low": 0.5,
    "high": 0.69
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.07369186284874256,
    "min": 0.07346713898004964,
    "max": 0.07402580551570281
   },
   "p95_request_seconds": {
    "mean": 0.0793910196904714,
    "min": 0.07851609797216952,
    "max": 0.08013061305973679
   },
   "cost_per_million_questions": null,
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.7406595665216446,
    "min": 0.7406595665216446,
    "max": 0.7406595665216446
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": null,
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.66,
     0.32,
     0.02,
     0.0
    ],
    "answers": 300,
    "yes_share": 0.7,
    "first_share_other": 0.6,
    "baseline": 0.5609673202614379,
    "above": 0.03903267973856206,
    "p": 0.194728946618818,
    "first_in_10": 7,
    "confidence": {
     "answers": 300,
     "mean_right": 0.7818167751034101,
     "mean_wrong": 0.6789237536489964,
     "sure": 0,
     "sure_wrong": 0
    },
    "reversed": {
     "run_1_accuracy": 0.6,
     "accuracy": 0.57,
     "change": -0.03,
     "same_choice_as_run_1": 0.79,
     "runs_1_2_agree": 1.0,
     "errors": 0,
     "missing": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     3,
     3,
     3
    ],
    "hard_right_all_runs": 3,
    "weights": {
     "repo": "wayfind/metask-jev-4b-policy-mix",
     "revision": "ea20fe8",
     "committed": "2026-09-22",
     "days_before": 41
    }
   }
  },
  {
   "provider": "d1-server",
   "model": "LiquidAI/d1-2B",
   "perplexity_mode": null,
   "request_model": "LiquidAI/d1-2B",
   "group": "decision",
   "web": false,
   "price": null,
   "runs": [
    {
     "started": "2026-11-05T19:07:49+00:00",
     "ended": "2026-11-05T19:07:50+00:00",
     "questions": 100,
     "correct": 54,
     "accuracy": 0.54,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.010540613962803036,
     "p95_request_seconds": 0.011910684988833964,
     "input_tokens": 14496,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "LiquidAI/d1-2B"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6272482816819195,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T19:07:50+00:00",
     "ended": "2026-11-05T19:07:51+00:00",
     "questions": 100,
     "correct": 54,
     "accuracy": 0.54,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.01070517353946343,
     "p95_request_seconds": 0.011694988003000617,
     "input_tokens": 14496,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "LiquidAI/d1-2B"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6272482816819195,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T19:07:51+00:00",
     "ended": "2026-11-05T19:07:52+00:00",
     "questions": 100,
     "correct": 54,
     "accuracy": 0.54,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.01117252151016146,
     "p95_request_seconds": 0.012699228944256902,
     "input_tokens": 14496,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "LiquidAI/d1-2B"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6272482816819195,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.54,
    "min": 0.54,
    "max": 0.54
   },
   "accuracy_ci95": {
    "low": 0.44,
    "high": 0.64
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.010806103004142642,
    "min": 0.010540613962803036,
    "max": 0.01117252151016146
   },
   "p95_request_seconds": {
    "mean": 0.01210163397869716,
    "min": 0.011694988003000617,
    "max": 0.012699228944256902
   },
   "cost_per_million_questions": null,
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.6272482816819195,
    "min": 0.6272482816819195,
    "max": 0.6272482816819195
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": null,
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.55,
     0.37,
     0.07,
     0.01
    ],
    "answers": 300,
    "yes_share": 0.6166666666666667,
    "first_share_other": 0.45,
    "baseline": 0.500718954248366,
    "above": 0.03928104575163405,
    "p": 0.20713193662568496,
    "first_in_10": 6,
    "confidence": {
     "answers": 300,
     "mean_right": 0.6342780005323078,
     "mean_wrong": 0.6189960030314637,
     "sure": 0,
     "sure_wrong": 0
    },
    "reversed": {
     "run_1_accuracy": 0.54,
     "accuracy": 0.56,
     "change": 0.02,
     "same_choice_as_run_1": 0.91,
     "runs_1_2_agree": 1.0,
     "errors": 0,
     "missing": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     1,
     1,
     1
    ],
    "hard_right_all_runs": 1,
    "weights": {
     "repo": "LiquidAI/d1-2B",
     "revision": "10e7671",
     "committed": "2026-10-29",
     "days_before": 4
    }
   }
  },
  {
   "provider": "strands",
   "model": "strands-decider-2b",
   "perplexity_mode": null,
   "request_model": "strands-decider-2b",
   "group": "decision",
   "web": false,
   "price": null,
   "runs": [
    {
     "started": "2026-11-05T16:11:58+00:00",
     "ended": "2026-11-05T16:12:00+00:00",
     "questions": 100,
     "correct": 55,
     "accuracy": 0.55,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.02471369248814881,
     "p95_request_seconds": 0.02674832392949611,
     "input_tokens": 17251,
     "output_tokens": 100,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "strands-decider-2b"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.706631,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:12:01+00:00",
     "ended": "2026-11-05T16:12:03+00:00",
     "questions": 100,
     "correct": 55,
     "accuracy": 0.55,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.024118270550388843,
     "p95_request_seconds": 0.027860549045726657,
     "input_tokens": 17251,
     "output_tokens": 100,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "strands-decider-2b"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.706631,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:12:03+00:00",
     "ended": "2026-11-05T16:12:06+00:00",
     "questions": 100,
     "correct": 55,
     "accuracy": 0.55,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.024142547568771988,
     "p95_request_seconds": 0.030077217030338943,
     "input_tokens": 17251,
     "output_tokens": 100,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "strands-decider-2b"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.706631,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.55,
    "min": 0.55,
    "max": 0.55
   },
   "accuracy_ci95": {
    "low": 0.45,
    "high": 0.65
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.024324836869103212,
    "min": 0.024118270550388843,
    "max": 0.02471369248814881
   },
   "p95_request_seconds": {
    "mean": 0.02822869666852057,
    "min": 0.02674832392949611,
    "max": 0.030077217030338943
   },
   "cost_per_million_questions": null,
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.706631,
    "min": 0.706631,
    "max": 0.706631
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": null,
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.69,
     0.26,
     0.05,
     0.0
    ],
    "answers": 300,
    "yes_share": 0.7,
    "first_share_other": 0.675,
    "baseline": 0.5570849673202615,
    "above": -0.007084967320261426,
    "p": 0.5632274422388919,
    "first_in_10": 7,
    "confidence": {
     "answers": 300,
     "mean_right": 0.7344181818181819,
     "mean_wrong": 0.6726688888888889,
     "sure": 0,
     "sure_wrong": 0
    },
    "reversed": {
     "run_1_accuracy": 0.55,
     "accuracy": 0.45,
     "change": -0.1,
     "same_choice_as_run_1": 0.72,
     "runs_1_2_agree": 1.0,
     "errors": 0,
     "missing": 0
    },
    "p_holm": 1.0,
    "explained": true,
    "hard_right": [
     4,
     4,
     4
    ],
    "hard_right_all_runs": 4,
    "weights": {
     "repo": "StrandsAgents/strands-decider-2B-hobson-v21",
     "revision": "2b52a62",
     "committed": "2026-10-05",
     "days_before": 28
    }
   }
  },
  {
   "provider": "gliner",
   "model": "gliner2.5-multi-decide",
   "perplexity_mode": null,
   "request_model": "gliner2.5-multi-decide",
   "group": "decision",
   "web": false,
   "price": null,
   "runs": [
    {
     "started": "2026-11-05T19:08:48+00:00",
     "ended": "2026-11-05T19:08:49+00:00",
     "questions": 100,
     "correct": 41,
     "accuracy": 0.41,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.007857523974962533,
     "p95_request_seconds": 0.009675738052465022,
     "input_tokens": 5941,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "gliner2.5-multi-decide"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6248292136192322,
      "sure": 1,
      "sure_wrong": 1
     }
    },
    {
     "started": "2026-11-05T19:08:49+00:00",
     "ended": "2026-11-05T19:08:50+00:00",
     "questions": 100,
     "correct": 41,
     "accuracy": 0.41,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.007512476004194468,
     "p95_request_seconds": 0.008375559002161026,
     "input_tokens": 5941,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "gliner2.5-multi-decide"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6248292136192322,
      "sure": 1,
      "sure_wrong": 1
     }
    },
    {
     "started": "2026-11-05T19:08:50+00:00",
     "ended": "2026-11-05T19:08:50+00:00",
     "questions": 100,
     "correct": 41,
     "accuracy": 0.41,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.007452185498550534,
     "p95_request_seconds": 0.00842212897259742,
     "input_tokens": 5941,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "gliner2.5-multi-decide"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6248292136192322,
      "sure": 1,
      "sure_wrong": 1
     }
    }
   ],
   "accuracy": {
    "mean": 0.41,
    "min": 0.41,
    "max": 0.41
   },
   "accuracy_ci95": {
    "low": 0.32,
    "high": 0.51
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.007607395159235845,
    "min": 0.007452185498550534,
    "max": 0.007857523974962533
   },
   "p95_request_seconds": {
    "mean": 0.008824475342407823,
    "min": 0.008375559002161026,
    "max": 0.009675738052465022
   },
   "cost_per_million_questions": null,
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.6248292136192322,
    "min": 0.6248292136192322,
    "max": 0.6248292136192322
   },
   "sure_wrong": {
    "mean": 1,
    "min": 1,
    "max": 1
   },
   "sure": {
    "mean": 1,
    "min": 1,
    "max": 1
   },
   "cost_kind": null,
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.39,
     0.51,
     0.09,
     0.01
    ],
    "answers": 300,
    "yes_share": 0.36666666666666664,
    "first_share_other": 0.425,
    "baseline": 0.41706535947712414,
    "above": -0.007065359477124167,
    "p": 0.5586932677199878,
    "first_in_10": 4,
    "confidence": {
     "answers": 300,
     "mean_right": 0.6234698957059441,
     "mean_wrong": 0.625773824372534,
     "sure": 3,
     "sure_wrong": 3
    },
    "p_holm": 1.0,
    "explained": true,
    "hard_right": [
     2,
     2,
     2
    ],
    "hard_right_all_runs": 2,
    "weights": {
     "repo": "fastino/GLiNER2.5-multi-Decide",
     "revision": "a35a0cd",
     "committed": "2026-09-28",
     "days_before": 35
    }
   }
  },
  {
   "provider": "laya",
   "model": "laya",
   "perplexity_mode": null,
   "request_model": "laya",
   "group": "decision",
   "web": false,
   "price": null,
   "runs": [
    {
     "started": "2026-11-05T19:10:59+00:00",
     "ended": "2026-11-05T19:11:00+00:00",
     "questions": 100,
     "correct": 57,
     "accuracy": 0.57,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.010720285528805107,
     "p95_request_seconds": 0.012313778977841139,
     "input_tokens": 13584,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "laya-rl-agent"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.701536,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T19:11:00+00:00",
     "ended": "2026-11-05T19:11:01+00:00",
     "questions": 100,
     "correct": 57,
     "accuracy": 0.57,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.010657435574103147,
     "p95_request_seconds": 0.012068258016370237,
     "input_tokens": 13584,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "laya-rl-agent"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.701536,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T19:11:01+00:00",
     "ended": "2026-11-05T19:11:02+00:00",
     "questions": 100,
     "correct": 57,
     "accuracy": 0.57,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.010605135990772396,
     "p95_request_seconds": 0.01172605890315026,
     "input_tokens": 13584,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "laya-rl-agent"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.701536,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.57,
    "min": 0.57,
    "max": 0.57
   },
   "accuracy_ci95": {
    "low": 0.47,
    "high": 0.66
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.010660952364560217,
    "min": 0.010605135990772396,
    "max": 0.010720285528805107
   },
   "p95_request_seconds": {
    "mean": 0.012036031965787211,
    "min": 0.01172605890315026,
    "max": 0.012313778977841139
   },
   "cost_per_million_questions": null,
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.701536,
    "min": 0.701536,
    "max": 0.701536
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": null,
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.59,
     0.32,
     0.08,
     0.01
    ],
    "answers": 300,
    "yes_share": 0.6,
    "first_share_other": 0.575,
    "baseline": 0.517202614379085,
    "above": 0.05279738562091496,
    "p": 0.12791573928934286,
    "first_in_10": 6,
    "confidence": {
     "answers": 300,
     "mean_right": 0.7150368421052632,
     "mean_wrong": 0.683639534883721,
     "sure": 0,
     "sure_wrong": 0
    },
    "reversed": {
     "run_1_accuracy": 0.57,
     "accuracy": 0.56,
     "change": -0.01,
     "same_choice_as_run_1": 0.71,
     "runs_1_2_agree": 1.0,
     "errors": 0,
     "missing": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     2,
     2,
     2
    ],
    "hard_right_all_runs": 2,
    "weights": {
     "repo": "convaiinnovations/laya",
     "revision": "c5d7873",
     "committed": "2026-09-19",
     "days_before": 44
    }
   }
  },
  {
   "provider": "laya",
   "model": "laya-multilingual",
   "perplexity_mode": null,
   "request_model": "laya-multilingual",
   "group": "decision",
   "web": false,
   "price": null,
   "runs": [
    {
     "started": "2026-11-05T19:12:19+00:00",
     "ended": "2026-11-05T19:12:20+00:00",
     "questions": 100,
     "correct": 54,
     "accuracy": 0.54,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.00891600752947852,
     "p95_request_seconds": 0.010847102967090905,
     "input_tokens": 13884,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "laya-rl-agent"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7473,
      "sure": 2,
      "sure_wrong": 1
     }
    },
    {
     "started": "2026-11-05T19:12:20+00:00",
     "ended": "2026-11-05T19:12:21+00:00",
     "questions": 100,
     "correct": 54,
     "accuracy": 0.54,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.009019554476253688,
     "p95_request_seconds": 0.01045670104213059,
     "input_tokens": 13884,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "laya-rl-agent"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7473,
      "sure": 2,
      "sure_wrong": 1
     }
    },
    {
     "started": "2026-11-05T19:12:21+00:00",
     "ended": "2026-11-05T19:12:22+00:00",
     "questions": 100,
     "correct": 54,
     "accuracy": 0.54,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.00893175054807216,
     "p95_request_seconds": 0.010379594983533025,
     "input_tokens": 13884,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "laya-rl-agent"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7473,
      "sure": 2,
      "sure_wrong": 1
     }
    }
   ],
   "accuracy": {
    "mean": 0.54,
    "min": 0.54,
    "max": 0.54
   },
   "accuracy_ci95": {
    "low": 0.44,
    "high": 0.64
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.008955770851268122,
    "min": 0.00891600752947852,
    "max": 0.009019554476253688
   },
   "p95_request_seconds": {
    "mean": 0.01056113299758484,
    "min": 0.010379594983533025,
    "max": 0.010847102967090905
   },
   "cost_per_million_questions": null,
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.7473,
    "min": 0.7473,
    "max": 0.7473
   },
   "sure_wrong": {
    "mean": 1,
    "min": 1,
    "max": 1
   },
   "sure": {
    "mean": 2,
    "min": 2,
    "max": 2
   },
   "cost_kind": null,
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.59,
     0.3,
     0.09,
     0.02
    ],
    "answers": 300,
    "yes_share": 0.7,
    "first_share_other": 0.425,
    "baseline": 0.5578496732026144,
    "above": -0.017849673202614325,
    "p": 0.6565129104747234,
    "first_in_10": 6,
    "confidence": {
     "answers": 300,
     "mean_right": 0.7732481481481481,
     "mean_wrong": 0.7168391304347826,
     "sure": 6,
     "sure_wrong": 3
    },
    "reversed": {
     "run_1_accuracy": 0.54,
     "accuracy": 0.54,
     "change": 0.0,
     "same_choice_as_run_1": 0.78,
     "runs_1_2_agree": 1.0,
     "errors": 0,
     "missing": 0
    },
    "p_holm": 1.0,
    "explained": true,
    "hard_right": [
     3,
     3,
     3
    ],
    "hard_right_all_runs": 3,
    "weights": {
     "repo": "convaiinnovations/laya",
     "revision": "c5d7873",
     "committed": "2026-09-19",
     "note": "its multilingual/ folder",
     "days_before": 44
    }
   }
  },
  {
   "provider": "onejev",
   "model": "OmniJev/OneJev-27B",
   "perplexity_mode": null,
   "request_model": "OmniJev/OneJev-27B",
   "group": "decision",
   "web": false,
   "price": null,
   "runs": [
    {
     "started": "2026-11-05T20:12:29+00:00",
     "ended": "2026-11-05T20:12:38+00:00",
     "questions": 100,
     "correct": 45,
     "accuracy": 0.45,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.09351494407746941,
     "p95_request_seconds": 0.13148148101754487,
     "input_tokens": 20433,
     "output_tokens": 100,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "OmniJev/OneJev-27B"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.60158914,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T20:12:38+00:00",
     "ended": "2026-11-05T20:12:48+00:00",
     "questions": 100,
     "correct": 45,
     "accuracy": 0.45,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.09358580649131909,
     "p95_request_seconds": 0.13179338397458196,
     "input_tokens": 20433,
     "output_tokens": 100,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "OmniJev/OneJev-27B"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.60158914,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T20:12:48+00:00",
     "ended": "2026-11-05T20:12:58+00:00",
     "questions": 100,
     "correct": 45,
     "accuracy": 0.45,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.09355967695591971,
     "p95_request_seconds": 0.13186702597886324,
     "input_tokens": 20433,
     "output_tokens": 100,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "OmniJev/OneJev-27B"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.60158914,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.45,
    "min": 0.45,
    "max": 0.45
   },
   "accuracy_ci95": {
    "low": 0.35,
    "high": 0.55
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.09355347584156941,
    "min": 0.09351494407746941,
    "max": 0.09358580649131909
   },
   "p95_request_seconds": {
    "mean": 0.1317139636569967,
    "min": 0.13148148101754487,
    "max": 0.13186702597886324
   },
   "cost_per_million_questions": null,
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.60158914,
    "min": 0.60158914,
    "max": 0.60158914
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": null,
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.32,
     0.61,
     0.05,
     0.02
    ],
    "answers": 300,
    "yes_share": 0.31666666666666665,
    "first_share_other": 0.325,
    "baseline": 0.38858169934640524,
    "above": 0.06141830065359477,
    "p": 0.08616657901636593,
    "first_in_10": 3,
    "confidence": {
     "answers": 300,
     "mean_right": 0.5948442888888889,
     "mean_wrong": 0.6071076545454546,
     "sure": 0,
     "sure_wrong": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     0,
     0,
     0
    ],
    "hard_right_all_runs": 0,
    "weights": {
     "repo": "OmniJev/OneJev-27B",
     "revision": "a5e5f55",
     "committed": "2026-10-16",
     "days_before": 17
    }
   }
  },
  {
   "provider": "reax",
   "model": "reax13",
   "perplexity_mode": null,
   "request_model": "reax13",
   "group": "decision",
   "web": false,
   "price": null,
   "runs": [
    {
     "started": "2026-11-06T22:10:24+00:00",
     "ended": "2026-11-06T22:10:46+00:00",
     "questions": 100,
     "correct": 55,
     "accuracy": 0.55,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.19033720303559676,
     "p95_request_seconds": 0.33339854900259525,
     "input_tokens": 0,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "reax13"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6621115879872534,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-06T22:10:46+00:00",
     "ended": "2026-11-06T22:11:11+00:00",
     "questions": 100,
     "correct": 55,
     "accuracy": 0.55,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.23653235606616363,
     "p95_request_seconds": 0.36780022201128304,
     "input_tokens": 0,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "reax13"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6621115879872534,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-06T22:11:12+00:00",
     "ended": "2026-11-06T22:11:37+00:00",
     "questions": 100,
     "correct": 55,
     "accuracy": 0.55,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.2196072309743613,
     "p95_request_seconds": 0.36273562896531075,
     "input_tokens": 0,
     "output_tokens": 0,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "reax13"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6621115879872534,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.55,
    "min": 0.55,
    "max": 0.55
   },
   "accuracy_ci95": {
    "low": 0.45,
    "high": 0.65
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.21549226335870722,
    "min": 0.19033720303559676,
    "max": 0.23653235606616363
   },
   "p95_request_seconds": {
    "mean": 0.35464479999306303,
    "min": 0.33339854900259525,
    "max": 0.36780022201128304
   },
   "cost_per_million_questions": null,
   "host": "our GPU",
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.6621115879872534,
    "min": 0.6621115879872534,
    "max": 0.6621115879872534
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": null,
   "cost_note": null,
   "method_note": null,
   "habits": {
    "positions": [
     0.49,
     0.46,
     0.04,
     0.01
    ],
    "answers": 300,
    "yes_share": 0.5333333333333333,
    "first_share_other": 0.425,
    "baseline": 0.4803856209150327,
    "above": 0.06961437908496737,
    "p": 0.07758587647325599,
    "first_in_10": 5,
    "confidence": {
     "answers": 300,
     "mean_right": 0.6859834220955647,
     "mean_wrong": 0.632934901854873,
     "sure": 0,
     "sure_wrong": 0
    },
    "reversed": {
     "run_1_accuracy": 0.55,
     "accuracy": 0.47,
     "change": -0.08,
     "same_choice_as_run_1": 0.87,
     "runs_1_2_agree": 1.0,
     "errors": 0,
     "missing": 0
    },
    "p_holm": 1.0,
    "explained": false,
    "hard_right": [
     1,
     1,
     1
    ],
    "hard_right_all_runs": 1,
    "weights": {
     "repo": "decisionmodels/REAX-1.3",
     "revision": "fa4bf7e",
     "committed": "2026-11-03",
     "days_before": -1
    }
   }
  },
  {
   "provider": "openai",
   "model": "gpt-6-astra",
   "perplexity_mode": null,
   "request_model": "gpt-6-astra",
   "group": "chat",
   "web": false,
   "price": {
    "basis": "OpenAI's API list price",
    "source": "https://developers.openai.com/api/docs/pricing",
    "date": "2026-10-08",
    "input_per_million": 10.0,
    "per_request": 0.0,
    "output_per_million": 50.0,
    "applies_to": "billed",
    "per_search": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T23:31:03+00:00",
     "ended": "2026-11-05T23:41:50+00:00",
     "questions": 100,
     "correct": 75,
     "accuracy": 0.75,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 10.253515571472235,
     "p95_request_seconds": 27.550826746039093,
     "input_tokens": 455668,
     "output_tokens": 26524,
     "cost_usd": 4.55232,
     "cost_per_million_questions": 45523.2,
     "served_models": [],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.88169,
      "sure": 24,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T23:41:58+00:00",
     "ended": "2026-11-05T23:52:20+00:00",
     "questions": 100,
     "correct": 74,
     "accuracy": 0.74,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 10.430041794548742,
     "p95_request_seconds": 26.58903404395096,
     "input_tokens": 455678,
     "output_tokens": 24429,
     "cost_usd": 3.988022,
     "cost_per_million_questions": 39880.22,
     "served_models": [],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.88596,
      "sure": 27,
      "sure_wrong": 1
     }
    },
    {
     "started": "2026-11-05T23:52:30+00:00",
     "ended": "2026-11-06T00:02:59+00:00",
     "questions": 100,
     "correct": 77,
     "accuracy": 0.77,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 10.004314572550356,
     "p95_request_seconds": 25.395156476064585,
     "input_tokens": 455656,
     "output_tokens": 25212,
     "cost_usd": 3.905992,
     "cost_per_million_questions": 39059.92,
     "served_models": [],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.87338,
      "sure": 23,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.7533333333333333,
    "min": 0.74,
    "max": 0.77
   },
   "accuracy_ci95": {
    "low": 0.6733333333333333,
    "high": 0.83
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 10.229290646190444,
    "min": 10.004314572550356,
    "max": 10.430041794548742
   },
   "p95_request_seconds": {
    "mean": 26.51167242201821,
    "min": 25.395156476064585,
    "max": 27.550826746039093
   },
   "cost_per_million_questions": {
    "mean": 41487.78,
    "min": 39059.92,
    "max": 45523.2
   },
   "run": 277,
   "reasoning": "high",
   "access": "Codex CLI, ChatGPT subscription",
   "interface": "CLI",
   "billing": "subscription",
   "workers": 2,
   "best_setting": true,
   "confidence_kind": "stated",
   "mean_chosen_probability": {
    "mean": 0.8803433333333334,
    "min": 0.87338,
    "max": 0.88596
   },
   "sure_wrong": {
    "mean": 0.3333333333333333,
    "min": 0,
    "max": 1
   },
   "sure": {
    "mean": 24.666666666666668,
    "min": 23,
    "max": 27
   },
   "cost_kind": "API-equivalent estimate",
   "cost_note": null,
   "method_note": "the Codex CLI adds its own context items to every request: 4,278 to 4,284 more input tokens than OpenAI's Responses API took for the same question (257 there; 62 questions compared), counted in the cost"
  },
  {
   "provider": "openai",
   "model": "gpt-6.1-sol",
   "perplexity_mode": null,
   "request_model": "gpt-6.1-sol",
   "group": "chat",
   "web": false,
   "price": {
    "basis": "OpenAI's API list price",
    "source": "https://developers.openai.com/api/docs/pricing",
    "date": "2026-10-08",
    "input_per_million": 2.0,
    "per_request": 0.0,
    "output_per_million": 10.0,
    "applies_to": "billed",
    "per_search": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T16:35:15+00:00",
     "ended": "2026-11-05T16:51:34+00:00",
     "questions": 100,
     "correct": 74,
     "accuracy": 0.74,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 14.856592767464463,
     "p95_request_seconds": 45.360500292037614,
     "input_tokens": 460416,
     "output_tokens": 41861,
     "cost_usd": 1.2270836,
     "cost_per_million_questions": 12270.836000000001,
     "served_models": [],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.88029,
      "sure": 26,
      "sure_wrong": 1
     }
    },
    {
     "started": "2026-11-05T16:51:53+00:00",
     "ended": "2026-11-05T17:07:55+00:00",
     "questions": 100,
     "correct": 76,
     "accuracy": 0.76,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 16.471834168070927,
     "p95_request_seconds": 42.14650849008467,
     "input_tokens": 470153,
     "output_tokens": 42218,
     "cost_usd": 1.0409756,
     "cost_per_million_questions": 10409.756,
     "served_models": [],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.88817,
      "sure": 26,
      "sure_wrong": 1
     }
    },
    {
     "started": "2026-11-05T17:08:06+00:00",
     "ended": "2026-11-05T17:24:14+00:00",
     "questions": 100,
     "correct": 77,
     "accuracy": 0.77,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 15.667244370910339,
     "p95_request_seconds": 43.06843907001894,
     "input_tokens": 460553,
     "output_tokens": 44854,
     "cost_usd": 1.0274636,
     "cost_per_million_questions": 10274.635999999999,
     "served_models": [],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.88028,
      "sure": 31,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.7566666666666667,
    "min": 0.74,
    "max": 0.77
   },
   "accuracy_ci95": {
    "low": 0.6766666666666667,
    "high": 0.83
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 15.665223768815244,
    "min": 14.856592767464463,
    "max": 16.471834168070927
   },
   "p95_request_seconds": {
    "mean": 43.525149284047075,
    "min": 42.14650849008467,
    "max": 45.360500292037614
   },
   "cost_per_million_questions": {
    "mean": 10985.076,
    "min": 10274.635999999999,
    "max": 12270.836000000001
   },
   "run": 276,
   "reasoning": "high",
   "access": "Codex CLI, ChatGPT subscription",
   "interface": "CLI",
   "billing": "subscription",
   "workers": 2,
   "best_setting": true,
   "confidence_kind": "stated",
   "mean_chosen_probability": {
    "mean": 0.8829133333333333,
    "min": 0.88028,
    "max": 0.88817
   },
   "sure_wrong": {
    "mean": 0.6666666666666666,
    "min": 0,
    "max": 1
   },
   "sure": {
    "mean": 27.666666666666668,
    "min": 26,
    "max": 31
   },
   "cost_kind": "API-equivalent estimate",
   "cost_note": null,
   "method_note": "the Codex CLI adds its own context items to every request: 4,278 to 4,284 more input tokens than OpenAI's Responses API took for the same question (257 there; 62 questions compared), counted in the cost"
  },
  {
   "provider": "openai",
   "model": "gpt-6.1-terra",
   "perplexity_mode": null,
   "request_model": "gpt-6.1-terra",
   "group": "chat",
   "web": false,
   "price": {
    "basis": "OpenAI's API list price",
    "source": "https://developers.openai.com/api/docs/pricing",
    "date": "2026-10-28",
    "input_per_million": 1.0,
    "per_request": 0.0,
    "output_per_million": 5.0,
    "applies_to": "billed",
    "per_search": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T17:24:21+00:00",
     "ended": "2026-11-05T17:29:53+00:00",
     "questions": 100,
     "correct": 77,
     "accuracy": 0.77,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 6.318064320017584,
     "p95_request_seconds": 10.612176149035804,
     "input_tokens": 455690,
     "output_tokens": 14858,
     "cost_usd": 0.351228,
     "cost_per_million_questions": 3512.28,
     "served_models": [],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.8279,
      "sure": 11,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T17:30:05+00:00",
     "ended": "2026-11-05T17:35:37+00:00",
     "questions": 100,
     "correct": 74,
     "accuracy": 0.74,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 6.130822371458635,
     "p95_request_seconds": 9.864700893987902,
     "input_tokens": 455640,
     "output_tokens": 15179,
     "cost_usd": 0.340015,
     "cost_per_million_questions": 3400.15,
     "served_models": [],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.8305,
      "sure": 13,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T17:35:44+00:00",
     "ended": "2026-11-05T17:41:22+00:00",
     "questions": 100,
     "correct": 68,
     "accuracy": 0.68,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 6.201180923962966,
     "p95_request_seconds": 11.15283597400412,
     "input_tokens": 455668,
     "output_tokens": 15076,
     "cost_usd": 0.3165456,
     "cost_per_million_questions": 3165.4559999999997,
     "served_models": [],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.83,
      "sure": 13,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.73,
    "min": 0.68,
    "max": 0.77
   },
   "accuracy_ci95": {
    "low": 0.65,
    "high": 0.8066666666666668
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 6.216689205146395,
    "min": 6.130822371458635,
    "max": 6.318064320017584
   },
   "p95_request_seconds": {
    "mean": 10.543237672342608,
    "min": 9.864700893987902,
    "max": 11.15283597400412
   },
   "cost_per_million_questions": {
    "mean": 3359.2953333333335,
    "min": 3165.4559999999997,
    "max": 3512.28
   },
   "run": 275,
   "reasoning": "high",
   "access": "Codex CLI, ChatGPT subscription",
   "interface": "CLI",
   "billing": "subscription",
   "workers": 2,
   "best_setting": true,
   "confidence_kind": "stated",
   "mean_chosen_probability": {
    "mean": 0.8294666666666667,
    "min": 0.8279,
    "max": 0.8305
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 12.333333333333334,
    "min": 11,
    "max": 13
   },
   "cost_kind": "API-equivalent estimate",
   "cost_note": null,
   "method_note": "the Codex CLI adds its own context items to every request: 4,278 to 4,284 more input tokens than OpenAI's Responses API took for the same question (257 there; 62 questions compared), counted in the cost"
  },
  {
   "provider": "openai",
   "model": "gpt-6-luna",
   "perplexity_mode": null,
   "request_model": "gpt-6-luna",
   "group": "chat",
   "web": false,
   "price": {
    "basis": "OpenAI's API list price",
    "source": "https://developers.openai.com/api/docs/pricing",
    "date": "2026-10-08",
    "input_per_million": 0.1,
    "per_request": 0.0,
    "output_per_million": 0.5,
    "applies_to": "billed",
    "per_search": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T17:41:28+00:00",
     "ended": "2026-11-05T17:44:58+00:00",
     "questions": 100,
     "correct": 54,
     "accuracy": 0.54,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 3.975570004549809,
     "p95_request_seconds": 5.914590802974999,
     "input_tokens": 455684,
     "output_tokens": 2600,
     "cost_usd": 0.035982,
     "cost_per_million_questions": 359.82,
     "served_models": [],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9191,
      "sure": 26,
      "sure_wrong": 11
     }
    },
    {
     "started": "2026-11-05T17:45:05+00:00",
     "ended": "2026-11-05T17:48:47+00:00",
     "questions": 100,
     "correct": 41,
     "accuracy": 0.41,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 3.88351703051012,
     "p95_request_seconds": 5.503877337090671,
     "input_tokens": 455688,
     "output_tokens": 2600,
     "cost_usd": 0.02799904,
     "cost_per_million_questions": 279.9904,
     "served_models": [],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.8938,
      "sure": 30,
      "sure_wrong": 17
     }
    },
    {
     "started": "2026-11-05T17:48:51+00:00",
     "ended": "2026-11-05T17:52:32+00:00",
     "questions": 100,
     "correct": 41,
     "accuracy": 0.41,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 4.084685304027516,
     "p95_request_seconds": 6.493374509969726,
     "input_tokens": 460505,
     "output_tokens": 2959,
     "cost_usd": 0.02801512,
     "cost_per_million_questions": 280.1512,
     "served_models": [],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.8986999999999999,
      "sure": 26,
      "sure_wrong": 14
     }
    }
   ],
   "accuracy": {
    "mean": 0.4533333333333333,
    "min": 0.41,
    "max": 0.54
   },
   "accuracy_ci95": {
    "low": 0.37,
    "high": 0.54
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 3.981257446362482,
    "min": 3.88351703051012,
    "max": 4.084685304027516
   },
   "p95_request_seconds": {
    "mean": 5.970614216678466,
    "min": 5.503877337090671,
    "max": 6.493374509969726
   },
   "cost_per_million_questions": {
    "mean": 306.65386666666666,
    "min": 279.9904,
    "max": 359.82
   },
   "run": 133,
   "reasoning": "low",
   "access": "Codex CLI, ChatGPT subscription",
   "interface": "CLI",
   "billing": "subscription",
   "workers": 2,
   "best_setting": true,
   "confidence_kind": "stated",
   "mean_chosen_probability": {
    "mean": 0.9038666666666667,
    "min": 0.8938,
    "max": 0.9191
   },
   "sure_wrong": {
    "mean": 14,
    "min": 11,
    "max": 17
   },
   "sure": {
    "mean": 27.333333333333332,
    "min": 26,
    "max": 30
   },
   "cost_kind": "API-equivalent estimate",
   "cost_note": null,
   "method_note": "the Codex CLI adds its own context items to every request: 4,278 to 4,284 more input tokens than OpenAI's Responses API took for the same question (257 there; 62 questions compared), counted in the cost"
  },
  {
   "provider": "gemini",
   "model": "gemini-3.8-flash",
   "perplexity_mode": null,
   "request_model": "gemini-3.8-flash",
   "group": "chat",
   "web": false,
   "price": {
    "basis": "Google's Gemini API price, introductory",
    "source": "https://ai.google.dev/gemini-api/docs/pricing",
    "date": "2026-10-08",
    "input_per_million": 0.75,
    "per_request": 0.0,
    "output_per_million": 3.75,
    "applies_to": "billed",
    "per_search": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T16:24:10+00:00",
     "ended": "2026-11-05T16:42:47+00:00",
     "questions": 100,
     "correct": 43,
     "accuracy": 0.43,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 22.55425359203946,
     "p95_request_seconds": 156.82007330702618,
     "input_tokens": 23121,
     "output_tokens": 534890,
     "cost_usd": 2.02317825,
     "cost_per_million_questions": 20231.7825,
     "served_models": [
      "gemini-3.8-flash"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9346,
      "sure": 13,
      "sure_wrong": 4
     }
    },
    {
     "started": "2026-11-05T16:45:15+00:00",
     "ended": "2026-11-05T17:03:42+00:00",
     "questions": 100,
     "correct": 47,
     "accuracy": 0.47,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 21.56652722094441,
     "p95_request_seconds": 162.43338739406317,
     "input_tokens": 23121,
     "output_tokens": 534601,
     "cost_usd": 2.0220945,
     "cost_per_million_questions": 20220.945,
     "served_models": [
      "gemini-3.8-flash"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9384,
      "sure": 15,
      "sure_wrong": 7
     }
    },
    {
     "started": "2026-11-05T17:05:28+00:00",
     "ended": "2026-11-05T17:23:33+00:00",
     "questions": 100,
     "correct": 48,
     "accuracy": 0.48,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 23.446750360541046,
     "p95_request_seconds": 183.03127926494926,
     "input_tokens": 23121,
     "output_tokens": 515449,
     "cost_usd": 1.9502745,
     "cost_per_million_questions": 19502.745,
     "served_models": [
      "gemini-3.8-flash"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9357,
      "sure": 18,
      "sure_wrong": 5
     }
    }
   ],
   "accuracy": {
    "mean": 0.45999999999999996,
    "min": 0.43,
    "max": 0.48
   },
   "accuracy_ci95": {
    "low": 0.37,
    "high": 0.5533333333333333
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 22.522510391174972,
    "min": 21.56652722094441,
    "max": 23.446750360541046
   },
   "p95_request_seconds": {
    "mean": 167.42824665534621,
    "min": 156.82007330702618,
    "max": 183.03127926494926
   },
   "cost_per_million_questions": {
    "mean": 19985.1575,
    "min": 19502.745,
    "max": 20231.7825
   },
   "run": 140,
   "reasoning": "high",
   "access": "Gemini API",
   "interface": null,
   "billing": "api",
   "workers": 4,
   "best_setting": true,
   "confidence_kind": "stated",
   "mean_chosen_probability": {
    "mean": 0.9362333333333334,
    "min": 0.9346,
    "max": 0.9384
   },
   "sure_wrong": {
    "mean": 5.333333333333333,
    "min": 4,
    "max": 7
   },
   "sure": {
    "mean": 15.333333333333334,
    "min": 13,
    "max": 18
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null
  },
  {
   "provider": "gemini",
   "model": "gemini-4-argon",
   "perplexity_mode": null,
   "request_model": "gemini-4-argon",
   "group": "chat",
   "web": false,
   "price": {
    "basis": "Google's Gemini API regular price (from 1 December 2026)",
    "source": "https://ai.google.dev/gemini-api/docs/pricing",
    "date": "2026-10-28",
    "input_per_million": 4.0,
    "per_request": 0.0,
    "output_per_million": 20.0,
    "applies_to": "billed",
    "per_search": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T17:26:08+00:00",
     "ended": "2026-11-05T17:28:03+00:00",
     "questions": 100,
     "correct": 67,
     "accuracy": 0.67,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 3.5002477270318195,
     "p95_request_seconds": 13.210390437045135,
     "input_tokens": 53871,
     "output_tokens": 3660,
     "cost_usd": 0.288684,
     "cost_per_million_questions": 2886.8399999999997,
     "served_models": [
      "gemini-4-argon"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7158,
      "sure": 12,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T17:28:25+00:00",
     "ended": "2026-11-05T17:30:45+00:00",
     "questions": 100,
     "correct": 70,
     "accuracy": 0.7,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 3.5046733479830436,
     "p95_request_seconds": 16.855469460017048,
     "input_tokens": 53871,
     "output_tokens": 3678,
     "cost_usd": 0.289044,
     "cost_per_million_questions": 2890.44,
     "served_models": [
      "gemini-4-argon"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.723,
      "sure": 9,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T17:30:54+00:00",
     "ended": "2026-11-05T17:32:34+00:00",
     "questions": 100,
     "correct": 69,
     "accuracy": 0.69,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 3.448258512013126,
     "p95_request_seconds": 8.262425886001438,
     "input_tokens": 53871,
     "output_tokens": 3637,
     "cost_usd": 0.288224,
     "cost_per_million_questions": 2882.24,
     "served_models": [
      "gemini-4-argon"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.714,
      "sure": 8,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.6866666666666666,
    "min": 0.67,
    "max": 0.7
   },
   "accuracy_ci95": {
    "low": 0.6,
    "high": 0.77
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 3.4843931956759966,
    "min": 3.448258512013126,
    "max": 3.5046733479830436
   },
   "p95_request_seconds": {
    "mean": 12.776095261021206,
    "min": 8.262425886001438,
    "max": 16.855469460017048
   },
   "cost_per_million_questions": {
    "mean": 2886.5066666666667,
    "min": 2882.24,
    "max": 2890.44
   },
   "run": 557,
   "reasoning": "low",
   "access": "Gemini API",
   "interface": null,
   "billing": "api",
   "workers": 4,
   "best_setting": true,
   "confidence_kind": "stated",
   "mean_chosen_probability": {
    "mean": 0.7176,
    "min": 0.714,
    "max": 0.723
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 9.666666666666666,
    "min": 8,
    "max": 12
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null
  },
  {
   "provider": "claude",
   "model": "claude-fable-5-1",
   "perplexity_mode": null,
   "request_model": "claude-fable-5-1",
   "group": "chat",
   "web": false,
   "price": {
    "basis": "Anthropic's API list price",
    "source": "https://platform.claude.com/docs/en/about-claude/pricing",
    "date": "2026-10-28",
    "input_per_million": 10.0,
    "per_request": 0.0,
    "output_per_million": 50.0,
    "applies_to": "billed",
    "per_search": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T16:24:11+00:00",
     "ended": "2026-11-05T16:28:04+00:00",
     "questions": 100,
     "correct": 71,
     "accuracy": 0.71,
     "missing": 0,
     "not_success": 2,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 8.668377798516303,
     "p95_request_seconds": 18.217831404996105,
     "input_tokens": 156709,
     "output_tokens": 51025,
     "cost_usd": 4.443238,
     "cost_per_million_questions": 44432.38,
     "served_models": [
      "claude-fable-5-1",
      "claude-opus-4-8"
     ],
     "re_asked": 0,
     "stand_in_refusals": 2,
     "confidence": {
      "answers": 98,
      "mean_chosen_probability": 0.7040816326530612,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:28:11+00:00",
     "ended": "2026-11-05T16:31:56+00:00",
     "questions": 100,
     "correct": 76,
     "accuracy": 0.76,
     "missing": 0,
     "not_success": 1,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 7.69301613693824,
     "p95_request_seconds": 19.320314334938303,
     "input_tokens": 153363,
     "output_tokens": 52868,
     "cost_usd": 3.38721925,
     "cost_per_million_questions": 33872.1925,
     "served_models": [
      "claude-fable-5-1",
      "claude-opus-4-8"
     ],
     "re_asked": 0,
     "stand_in_refusals": 1,
     "confidence": {
      "answers": 99,
      "mean_chosen_probability": 0.6992929292929293,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:32:01+00:00",
     "ended": "2026-11-05T16:35:35+00:00",
     "questions": 100,
     "correct": 74,
     "accuracy": 0.74,
     "missing": 0,
     "not_success": 2,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 7.277771000051871,
     "p95_request_seconds": 18.850650737062097,
     "input_tokens": 156700,
     "output_tokens": 48881,
     "cost_usd": 3.20154125,
     "cost_per_million_questions": 32015.4125,
     "served_models": [
      "claude-fable-5-1",
      "claude-opus-4-8"
     ],
     "re_asked": 0,
     "stand_in_refusals": 2,
     "confidence": {
      "answers": 98,
      "mean_chosen_probability": 0.7037755102040816,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.7366666666666667,
    "min": 0.71,
    "max": 0.76
   },
   "accuracy_ci95": {
    "low": 0.66,
    "high": 0.81
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 7.879721645168805,
    "min": 7.277771000051871,
    "max": 8.668377798516303
   },
   "p95_request_seconds": {
    "mean": 18.796265492332168,
    "min": 18.217831404996105,
    "max": 19.320314334938303
   },
   "cost_per_million_questions": {
    "mean": 36773.32833333333,
    "min": 32015.4125,
    "max": 44432.38
   },
   "run": 310,
   "reasoning": "high",
   "access": "Claude Code CLI, Claude subscription",
   "interface": "CLI",
   "billing": "subscription",
   "workers": 4,
   "best_setting": true,
   "confidence_kind": "stated",
   "mean_chosen_probability": {
    "mean": 0.7023833573833573,
    "min": 0.6992929292929293,
    "max": 0.7040816326530612
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "API-equivalent estimate",
   "cost_note": null,
   "method_note": "Claude Code writes the prompt to its 1-hour cache, counted in the cost",
   "stand_in": {
    "models": [
     "claude-opus-4-8"
    ],
    "requests": [
     2,
     1,
     2
    ],
    "questions": 2,
    "scored": "as failures (refusals), not with the other model's answers"
   }
  },
  {
   "provider": "claude",
   "model": "claude-opus-5-5",
   "perplexity_mode": null,
   "request_model": "claude-opus-5-5",
   "group": "chat",
   "web": false,
   "price": {
    "basis": "Anthropic's API list price",
    "source": "https://platform.claude.com/docs/en/about-claude/pricing",
    "date": "2026-10-08",
    "input_per_million": 4.0,
    "per_request": 0.0,
    "output_per_million": 20.0,
    "applies_to": "billed",
    "per_search": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T16:35:47+00:00",
     "ended": "2026-11-05T16:37:58+00:00",
     "questions": 100,
     "correct": 81,
     "accuracy": 0.81,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 4.9958457665052265,
     "p95_request_seconds": 9.404322471935302,
     "input_tokens": 148022,
     "output_tokens": 32304,
     "cost_usd": 1.3688504,
     "cost_per_million_questions": 13688.503999999999,
     "served_models": [
      "claude-opus-5-5"
     ],
     "re_asked": 0,
     "stand_in_refusals": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7121,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:38:05+00:00",
     "ended": "2026-11-05T16:40:22+00:00",
     "questions": 100,
     "correct": 78,
     "accuracy": 0.78,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 5.08507604902843,
     "p95_request_seconds": 9.416104063973762,
     "input_tokens": 148041,
     "output_tokens": 33985,
     "cost_usd": 0.9510258,
     "cost_per_million_questions": 9510.258,
     "served_models": [
      "claude-opus-5-5"
     ],
     "re_asked": 0,
     "stand_in_refusals": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7085,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:40:27+00:00",
     "ended": "2026-11-05T16:42:50+00:00",
     "questions": 100,
     "correct": 77,
     "accuracy": 0.77,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 5.488293179078028,
     "p95_request_seconds": 9.115376226021908,
     "input_tokens": 148023,
     "output_tokens": 34546,
     "cost_usd": 0.9621018,
     "cost_per_million_questions": 9621.018,
     "served_models": [
      "claude-opus-5-5"
     ],
     "re_asked": 0,
     "stand_in_refusals": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7076,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.7866666666666667,
    "min": 0.77,
    "max": 0.81
   },
   "accuracy_ci95": {
    "low": 0.7066666666666667,
    "high": 0.86
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 5.189738331537228,
    "min": 4.9958457665052265,
    "max": 5.488293179078028
   },
   "p95_request_seconds": {
    "mean": 9.311934253976991,
    "min": 9.115376226021908,
    "max": 9.416104063973762
   },
   "cost_per_million_questions": {
    "mean": 10939.926666666666,
    "min": 9510.258,
    "max": 13688.503999999999
   },
   "run": 278,
   "reasoning": "high",
   "access": "Claude Code CLI, Claude subscription",
   "interface": "CLI",
   "billing": "subscription",
   "workers": 4,
   "best_setting": true,
   "confidence_kind": "stated",
   "mean_chosen_probability": {
    "mean": 0.7094,
    "min": 0.7076,
    "max": 0.7121
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "API-equivalent estimate",
   "cost_note": null,
   "method_note": "Claude Code writes the prompt to its 1-hour cache, counted in the cost"
  },
  {
   "provider": "claude",
   "model": "claude-sonnet-5-5",
   "perplexity_mode": null,
   "request_model": "claude-sonnet-5-5",
   "group": "chat",
   "web": false,
   "price": {
    "basis": "Anthropic's API list price",
    "source": "https://platform.claude.com/docs/en/about-claude/pricing",
    "date": "2026-10-08",
    "input_per_million": 2.0,
    "per_request": 0.0,
    "output_per_million": 10.0,
    "applies_to": "billed",
    "per_search": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T16:42:54+00:00",
     "ended": "2026-11-05T16:45:26+00:00",
     "questions": 100,
     "correct": 77,
     "accuracy": 0.77,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 5.6496202779817395,
     "p95_request_seconds": 11.01767991494853,
     "input_tokens": 148423,
     "output_tokens": 36398,
     "cost_usd": 0.7152224,
     "cost_per_million_questions": 7152.224,
     "served_models": [
      "claude-sonnet-5-5"
     ],
     "re_asked": 0,
     "stand_in_refusals": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7151,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:45:34+00:00",
     "ended": "2026-11-05T16:47:48+00:00",
     "questions": 100,
     "correct": 80,
     "accuracy": 0.8,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 5.230450542934705,
     "p95_request_seconds": 9.277064668014646,
     "input_tokens": 148416,
     "output_tokens": 37700,
     "cost_usd": 0.5141629,
     "cost_per_million_questions": 5141.629,
     "served_models": [
      "claude-sonnet-5-5"
     ],
     "re_asked": 0,
     "stand_in_refusals": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.713,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:47:53+00:00",
     "ended": "2026-11-05T16:50:13+00:00",
     "questions": 100,
     "correct": 83,
     "accuracy": 0.83,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 5.225340876437258,
     "p95_request_seconds": 9.725356057984754,
     "input_tokens": 148432,
     "output_tokens": 36803,
     "cost_usd": 0.5052569,
     "cost_per_million_questions": 5052.5689999999995,
     "served_models": [
      "claude-sonnet-5-5"
     ],
     "re_asked": 0,
     "stand_in_refusals": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7138,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.8,
    "min": 0.77,
    "max": 0.83
   },
   "accuracy_ci95": {
    "low": 0.7266666666666667,
    "high": 0.8666666666666667
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 5.368470565784567,
    "min": 5.225340876437258,
    "max": 5.6496202779817395
   },
   "p95_request_seconds": {
    "mean": 10.00670021364931,
    "min": 9.277064668014646,
    "max": 11.01767991494853
   },
   "cost_per_million_questions": {
    "mean": 5782.140666666666,
    "min": 5052.5689999999995,
    "max": 7152.224
   },
   "run": 193,
   "reasoning": "high",
   "access": "Claude Code CLI, Claude subscription",
   "interface": "CLI",
   "billing": "subscription",
   "workers": 4,
   "best_setting": true,
   "confidence_kind": "stated",
   "mean_chosen_probability": {
    "mean": 0.7139666666666666,
    "min": 0.713,
    "max": 0.7151
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "API-equivalent estimate",
   "cost_note": null,
   "method_note": "Claude Code writes the prompt to its 1-hour cache, counted in the cost"
  },
  {
   "provider": "xai",
   "model": "grok-4.7",
   "perplexity_mode": null,
   "request_model": "grok-4.7",
   "group": "chat",
   "web": false,
   "price": {
    "basis": "xAI's API list price",
    "source": "https://docs.x.ai/docs/models/grok-4.7",
    "date": "2026-10-28",
    "input_per_million": 2.0,
    "per_request": 0.0,
    "output_per_million": 6.0,
    "applies_to": "billed",
    "per_search": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T16:24:10+00:00",
     "ended": "2026-11-05T17:06:18+00:00",
     "questions": 100,
     "correct": 49,
     "accuracy": 0.49,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 1,
     "median_request_seconds": 55.69117592694238,
     "p95_request_seconds": 347.7815774679184,
     "input_tokens": 158547,
     "output_tokens": 565093,
     "cost_usd": 3.518532,
     "cost_per_million_questions": 35185.32,
     "served_models": [
      "grok-4.7"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6498,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T17:08:21+00:00",
     "ended": "2026-11-05T17:49:27+00:00",
     "questions": 100,
     "correct": 52,
     "accuracy": 0.52,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 67.04504711797927,
     "p95_request_seconds": 320.799600418075,
     "input_tokens": 158547,
     "output_tokens": 572480,
     "cost_usd": 3.55287,
     "cost_per_million_questions": 35528.7,
     "served_models": [
      "grok-4.7"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6584,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T17:54:11+00:00",
     "ended": "2026-11-05T18:35:01+00:00",
     "questions": 100,
     "correct": 50,
     "accuracy": 0.5,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 50.653262732492294,
     "p95_request_seconds": 362.754464650061,
     "input_tokens": 158547,
     "output_tokens": 588540,
     "cost_usd": 3.652494,
     "cost_per_million_questions": 36524.94,
     "served_models": [
      "grok-4.7"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.6602,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.5033333333333333,
    "min": 0.49,
    "max": 0.52
   },
   "accuracy_ci95": {
    "low": 0.41666666666666663,
    "high": 0.5933333333333334
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 57.79649525913798,
    "min": 50.653262732492294,
    "max": 67.04504711797927
   },
   "p95_request_seconds": {
    "mean": 343.7785475120181,
    "min": 320.799600418075,
    "max": 362.754464650061
   },
   "cost_per_million_questions": {
    "mean": 35746.32,
    "min": 35185.32,
    "max": 36524.94
   },
   "run": 311,
   "reasoning": "high",
   "access": "xAI Responses API, SuperGrok subscription",
   "interface": null,
   "billing": "subscription",
   "workers": 4,
   "best_setting": true,
   "confidence_kind": "stated",
   "mean_chosen_probability": {
    "mean": 0.6561333333333333,
    "min": 0.6498,
    "max": 0.6602
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "API-equivalent estimate",
   "cost_note": null,
   "method_note": null
  },
  {
   "provider": "openrouter-chat",
   "model": "moonshotai/kimi-k3.1-pro",
   "perplexity_mode": null,
   "request_model": "moonshotai/kimi-k3.1-pro",
   "group": "chat",
   "web": false,
   "price": {
    "basis": "OpenRouter's price for Moonshot AI's endpoint (mxfp4)",
    "source": "https://openrouter.ai/moonshotai/kimi-k3.1-pro",
    "date": "2026-10-28",
    "input_per_million": 2.0,
    "per_request": 0.0,
    "output_per_million": 10.0,
    "applies_to": "billed",
    "per_search": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T16:24:10+00:00",
     "ended": "2026-11-05T16:26:39+00:00",
     "questions": 100,
     "correct": 66,
     "accuracy": 0.66,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 5.215241002035327,
     "p95_request_seconds": 11.651195683982223,
     "input_tokens": 45148,
     "output_tokens": 15066,
     "cost_usd": 0.240956,
     "cost_per_million_questions": 2409.56,
     "served_models": [
      "moonshotai/kimi-k3.1-pro"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9114,
      "sure": 20,
      "sure_wrong": 3
     }
    },
    {
     "started": "2026-11-05T16:26:43+00:00",
     "ended": "2026-11-05T16:29:16+00:00",
     "questions": 100,
     "correct": 66,
     "accuracy": 0.66,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 5.396328423987143,
     "p95_request_seconds": 10.802172562922351,
     "input_tokens": 45148,
     "output_tokens": 13899,
     "cost_usd": 0.1786034,
     "cost_per_million_questions": 1786.0339999999999,
     "served_models": [
      "moonshotai/kimi-k3.1-pro"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9234,
      "sure": 26,
      "sure_wrong": 7
     }
    },
    {
     "started": "2026-11-05T16:29:20+00:00",
     "ended": "2026-11-05T16:31:56+00:00",
     "questions": 100,
     "correct": 67,
     "accuracy": 0.67,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 5.323983088019304,
     "p95_request_seconds": 11.78730937000364,
     "input_tokens": 45148,
     "output_tokens": 13890,
     "cost_usd": 0.1785134,
     "cost_per_million_questions": 1785.1339999999998,
     "served_models": [
      "moonshotai/kimi-k3.1-pro"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9166,
      "sure": 23,
      "sure_wrong": 7
     }
    }
   ],
   "accuracy": {
    "mean": 0.6633333333333333,
    "min": 0.66,
    "max": 0.67
   },
   "accuracy_ci95": {
    "low": 0.5866666666666667,
    "high": 0.7366666666666667
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 5.311850838013925,
    "min": 5.215241002035327,
    "max": 5.396328423987143
   },
   "p95_request_seconds": {
    "mean": 11.413559205636071,
    "min": 10.802172562922351,
    "max": 11.78730937000364
   },
   "cost_per_million_questions": {
    "mean": 1993.5759999999998,
    "min": 1785.1339999999998,
    "max": 2409.56
   },
   "run": 320,
   "reasoning": "low",
   "access": "OpenRouter chat API, Moonshot AI's endpoint (mxfp4), fallbacks off",
   "interface": null,
   "billing": "api",
   "workers": 4,
   "best_setting": true,
   "confidence_kind": "stated",
   "mean_chosen_probability": {
    "mean": 0.9171333333333334,
    "min": 0.9114,
    "max": 0.9234
   },
   "sure_wrong": {
    "mean": 5.666666666666667,
    "min": 3,
    "max": 7
   },
   "sure": {
    "mean": 23,
    "min": 20,
    "max": 26
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null
  },
  {
   "provider": "openrouter-chat",
   "model": "z-ai/glm-5.3",
   "perplexity_mode": null,
   "request_model": "z-ai/glm-5.3",
   "group": "chat",
   "web": false,
   "price": {
    "basis": "OpenRouter's price for Fireworks' spot endpoint, which served nearly every request",
    "source": "https://openrouter.ai/z-ai/glm-5.3",
    "date": "2026-10-28",
    "input_per_million": 0.98,
    "per_request": 0.0,
    "output_per_million": 3.08,
    "applies_to": "billed",
    "per_search": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T16:24:10+00:00",
     "ended": "2026-11-05T16:24:48+00:00",
     "questions": 100,
     "correct": 61,
     "accuracy": 0.61,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 1.4507736804662272,
     "p95_request_seconds": 2.3277657569851726,
     "input_tokens": 22614,
     "output_tokens": 6780,
     "cost_usd": 0.035225316,
     "cost_per_million_questions": 352.25316,
     "served_models": [
      "z-ai/glm-5.3"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7053,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:24:50+00:00",
     "ended": "2026-11-05T16:25:27+00:00",
     "questions": 100,
     "correct": 58,
     "accuracy": 0.58,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 1.397353503445629,
     "p95_request_seconds": 2.5689726000418887,
     "input_tokens": 22614,
     "output_tokens": 6920,
     "cost_usd": 0.026998216000000002,
     "cost_per_million_questions": 269.98216,
     "served_models": [
      "z-ai/glm-5.3"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7326,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:25:28+00:00",
     "ended": "2026-11-05T16:26:06+00:00",
     "questions": 100,
     "correct": 64,
     "accuracy": 0.64,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 1.3873360950383358,
     "p95_request_seconds": 2.503167246002704,
     "input_tokens": 22614,
     "output_tokens": 7235,
     "cost_usd": 0.027523132,
     "cost_per_million_questions": 275.23132,
     "served_models": [
      "z-ai/glm-5.3"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7266,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.61,
    "min": 0.58,
    "max": 0.64
   },
   "accuracy_ci95": {
    "low": 0.53,
    "high": 0.6833333333333332
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 1.4118210929833974,
    "min": 1.3873360950383358,
    "max": 1.4507736804662272
   },
   "p95_request_seconds": {
    "mean": 2.4666352010099217,
    "min": 2.3277657569851726,
    "max": 2.5689726000418887
   },
   "cost_per_million_questions": {
    "mean": 299.1555466666667,
    "min": 269.98216,
    "max": 352.25316
   },
   "run": 314,
   "reasoning": "low",
   "access": "OpenRouter chat API, Fireworks endpoint, fallbacks off",
   "interface": null,
   "billing": "api",
   "workers": 4,
   "best_setting": true,
   "confidence_kind": "stated",
   "mean_chosen_probability": {
    "mean": 0.7215,
    "min": 0.7053,
    "max": 0.7326
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": null,
   "method_note": null
  },
  {
   "provider": "vllm",
   "model": "gemma-4-31B-it",
   "perplexity_mode": null,
   "request_model": "gemma-4-31B-it",
   "group": "chat",
   "web": false,
   "price": null,
   "runs": [
    {
     "started": "2026-11-05T20:10:25+00:00",
     "ended": "2026-11-05T20:10:34+00:00",
     "questions": 100,
     "correct": 54,
     "accuracy": 0.54,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.35101353854406625,
     "p95_request_seconds": 0.3885917909210548,
     "input_tokens": 23921,
     "output_tokens": 1099,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "gemma-4-31B-it"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9706764119781285,
      "sure": 75,
      "sure_wrong": 32
     }
    },
    {
     "started": "2026-11-05T20:10:34+00:00",
     "ended": "2026-11-05T20:10:41+00:00",
     "questions": 100,
     "correct": 54,
     "accuracy": 0.54,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.30640298448270187,
     "p95_request_seconds": 0.3077819790923968,
     "input_tokens": 23921,
     "output_tokens": 1099,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "gemma-4-31B-it"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9710944754938976,
      "sure": 74,
      "sure_wrong": 31
     }
    },
    {
     "started": "2026-11-05T20:10:42+00:00",
     "ended": "2026-11-05T20:10:49+00:00",
     "questions": 100,
     "correct": 54,
     "accuracy": 0.54,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.3062934284680523,
     "p95_request_seconds": 0.30829010903835297,
     "input_tokens": 23921,
     "output_tokens": 1099,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "gemma-4-31B-it"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9710944754938976,
      "sure": 74,
      "sure_wrong": 31
     }
    }
   ],
   "accuracy": {
    "mean": 0.54,
    "min": 0.54,
    "max": 0.54
   },
   "accuracy_ci95": {
    "low": 0.44,
    "high": 0.64
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.3212366504982735,
    "min": 0.3062934284680523,
    "max": 0.35101353854406625
   },
   "p95_request_seconds": {
    "mean": 0.33488795968393487,
    "min": 0.3077819790923968,
    "max": 0.3885917909210548
   },
   "cost_per_million_questions": null,
   "host": "our GPU",
   "run": 119,
   "reasoning": "off",
   "access": "vLLM, self-hosted, FP8",
   "interface": null,
   "billing": "local",
   "workers": 4,
   "best_setting": true,
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.9709551209886412,
    "min": 0.9706764119781285,
    "max": 0.9710944754938976
   },
   "sure_wrong": {
    "mean": 31.333333333333332,
    "min": 31,
    "max": 32
   },
   "sure": {
    "mean": 74.33333333333333,
    "min": 74,
    "max": 75
   },
   "cost_kind": null,
   "cost_note": null,
   "method_note": null
  },
  {
   "provider": "vllm",
   "model": "qwen38-27b-bf16",
   "perplexity_mode": null,
   "request_model": "qwen38-27b-bf16",
   "group": "chat",
   "web": false,
   "price": null,
   "runs": [
    {
     "started": "2026-11-05T19:21:29+00:00",
     "ended": "2026-11-05T19:24:21+00:00",
     "questions": 100,
     "correct": 58,
     "accuracy": 0.58,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 5.630892608023714,
     "p95_request_seconds": 16.202080703922547,
     "input_tokens": 25647,
     "output_tokens": 41714,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "qwen38-27b-bf16"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9972115578373155,
      "sure": 95,
      "sure_wrong": 41
     }
    },
    {
     "started": "2026-11-05T19:24:42+00:00",
     "ended": "2026-11-05T19:27:27+00:00",
     "questions": 100,
     "correct": 57,
     "accuracy": 0.57,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 5.532156884029973,
     "p95_request_seconds": 16.10587900003884,
     "input_tokens": 25647,
     "output_tokens": 39453,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "qwen38-27b-bf16"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9983257244165674,
      "sure": 99,
      "sure_wrong": 43
     }
    },
    {
     "started": "2026-11-05T19:27:39+00:00",
     "ended": "2026-11-05T19:30:26+00:00",
     "questions": 100,
     "correct": 58,
     "accuracy": 0.58,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 5.746107511513401,
     "p95_request_seconds": 16.832469890010543,
     "input_tokens": 25647,
     "output_tokens": 40490,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "qwen38-27b-bf16"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9981613272583404,
      "sure": 97,
      "sure_wrong": 42
     }
    }
   ],
   "accuracy": {
    "mean": 0.5766666666666667,
    "min": 0.57,
    "max": 0.58
   },
   "accuracy_ci95": {
    "low": 0.48666666666666664,
    "high": 0.6633333333333333
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 5.636385667855696,
    "min": 5.532156884029973,
    "max": 5.746107511513401
   },
   "p95_request_seconds": {
    "mean": 16.380143197990645,
    "min": 16.10587900003884,
    "max": 16.832469890010543
   },
   "cost_per_million_questions": null,
   "host": "our GPU",
   "run": 318,
   "reasoning": "low",
   "access": "vLLM, self-hosted",
   "interface": null,
   "billing": "local",
   "workers": 4,
   "best_setting": true,
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.9978995365040745,
    "min": 0.9972115578373155,
    "max": 0.9983257244165674
   },
   "sure_wrong": {
    "mean": 42,
    "min": 41,
    "max": 43
   },
   "sure": {
    "mean": 97,
    "min": 95,
    "max": 99
   },
   "cost_kind": null,
   "cost_note": null,
   "method_note": null
  },
  {
   "provider": "vllm",
   "model": "qwen3-14b",
   "perplexity_mode": null,
   "request_model": "qwen3-14b",
   "group": "chat",
   "web": false,
   "price": null,
   "runs": [
    {
     "started": "2026-11-05T19:55:08+00:00",
     "ended": "2026-11-05T19:55:17+00:00",
     "questions": 100,
     "correct": 59,
     "accuracy": 0.59,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.3814098699949682,
     "p95_request_seconds": 0.4156967019662261,
     "input_tokens": 22523,
     "output_tokens": 1703,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "qwen3-14b"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9561719093796185,
      "sure": 68,
      "sure_wrong": 25
     }
    },
    {
     "started": "2026-11-05T19:55:17+00:00",
     "ended": "2026-11-05T19:55:26+00:00",
     "questions": 100,
     "correct": 59,
     "accuracy": 0.59,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.35919671948067844,
     "p95_request_seconds": 0.37871723796706647,
     "input_tokens": 22523,
     "output_tokens": 1703,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "qwen3-14b"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9577063554472206,
      "sure": 66,
      "sure_wrong": 24
     }
    },
    {
     "started": "2026-11-05T19:55:26+00:00",
     "ended": "2026-11-05T19:55:35+00:00",
     "questions": 100,
     "correct": 59,
     "accuracy": 0.59,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.35883679997641593,
     "p95_request_seconds": 0.3788583310088143,
     "input_tokens": 22523,
     "output_tokens": 1703,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "qwen3-14b"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9553221738420504,
      "sure": 68,
      "sure_wrong": 24
     }
    }
   ],
   "accuracy": {
    "mean": 0.59,
    "min": 0.59,
    "max": 0.59
   },
   "accuracy_ci95": {
    "low": 0.49,
    "high": 0.69
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.3664811298173542,
    "min": 0.35883679997641593,
    "max": 0.3814098699949682
   },
   "p95_request_seconds": {
    "mean": 0.3910907569807023,
    "min": 0.37871723796706647,
    "max": 0.4156967019662261
   },
   "cost_per_million_questions": null,
   "host": "our GPU",
   "run": 131,
   "reasoning": "off",
   "access": "vLLM, self-hosted, BF16",
   "interface": null,
   "billing": "local",
   "workers": 4,
   "best_setting": true,
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.9564001462229632,
    "min": 0.9553221738420504,
    "max": 0.9577063554472206
   },
   "sure_wrong": {
    "mean": 24.333333333333332,
    "min": 24,
    "max": 25
   },
   "sure": {
    "mean": 67.33333333333333,
    "min": 66,
    "max": 68
   },
   "cost_kind": null,
   "cost_note": null,
   "method_note": null
  },
  {
   "provider": "vllm",
   "model": "erk-14b",
   "perplexity_mode": null,
   "request_model": "erk-14b",
   "group": "chat",
   "web": false,
   "price": null,
   "runs": [
    {
     "started": "2026-11-05T19:42:16+00:00",
     "ended": "2026-11-05T19:42:23+00:00",
     "questions": 100,
     "correct": 65,
     "accuracy": 0.65,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.2714290829608217,
     "p95_request_seconds": 0.28381655900739133,
     "input_tokens": 22523,
     "output_tokens": 1096,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "erk-14b"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7828275101351622,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T19:42:23+00:00",
     "ended": "2026-11-05T19:42:29+00:00",
     "questions": 100,
     "correct": 65,
     "accuracy": 0.65,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.24622164299944416,
     "p95_request_seconds": 0.2478342039976269,
     "input_tokens": 22523,
     "output_tokens": 1098,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "erk-14b"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7826034132886089,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T19:42:29+00:00",
     "ended": "2026-11-05T19:42:35+00:00",
     "questions": 100,
     "correct": 65,
     "accuracy": 0.65,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.24648458749288693,
     "p95_request_seconds": 0.24810333200730383,
     "input_tokens": 22523,
     "output_tokens": 1098,
     "cost_usd": null,
     "cost_per_million_questions": null,
     "served_models": [
      "erk-14b"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.781405280080991,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.65,
    "min": 0.65,
    "max": 0.65
   },
   "accuracy_ci95": {
    "low": 0.55,
    "high": 0.74
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.2547117711510509,
    "min": 0.24622164299944416,
    "max": 0.2714290829608217
   },
   "p95_request_seconds": {
    "mean": 0.25991803167077404,
    "min": 0.2478342039976269,
    "max": 0.28381655900739133
   },
   "cost_per_million_questions": null,
   "host": "our GPU",
   "run": 132,
   "reasoning": "off",
   "access": "vLLM, self-hosted, BF16",
   "interface": null,
   "billing": "local",
   "workers": 4,
   "best_setting": true,
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.7822787345015874,
    "min": 0.781405280080991,
    "max": 0.7828275101351622
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": null,
   "cost_note": null,
   "method_note": null
  },
  {
   "provider": "vllm",
   "model": "deepseek-v4.1-flash-1017",
   "perplexity_mode": null,
   "request_model": "deepseek-v4.1-flash-1017",
   "group": "chat",
   "web": false,
   "price": {
    "basis": "DeepSeek's list price for the same model",
    "source": "https://api-docs.deepseek.com/quick_start/pricing",
    "date": "2026-11-05",
    "input_per_million": 0.3,
    "per_request": 0.0,
    "output_per_million": 1.2,
    "applies_to": "maker",
    "per_search": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T23:28:41+00:00",
     "ended": "2026-11-05T23:39:23+00:00",
     "questions": 100,
     "correct": 63,
     "accuracy": 0.63,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 18.741823738033418,
     "p95_request_seconds": 64.2766059130663,
     "input_tokens": 23548,
     "output_tokens": 101474,
     "cost_usd": 0.1288332,
     "cost_per_million_questions": 1288.332,
     "served_models": [
      "deepseek-v4.1-flash-1017"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9987798592538969,
      "sure": 99,
      "sure_wrong": 37
     }
    },
    {
     "started": "2026-11-05T23:39:40+00:00",
     "ended": "2026-11-05T23:50:13+00:00",
     "questions": 100,
     "correct": 58,
     "accuracy": 0.58,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 15.595324341964442,
     "p95_request_seconds": 71.95856384502258,
     "input_tokens": 23548,
     "output_tokens": 109285,
     "cost_usd": 0.13820639999999998,
     "cost_per_million_questions": 1382.0639999999996,
     "served_models": [
      "deepseek-v4.1-flash-1017"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9997219382752126,
      "sure": 100,
      "sure_wrong": 42
     }
    },
    {
     "started": "2026-11-05T23:50:45+00:00",
     "ended": "2026-11-06T00:01:25+00:00",
     "questions": 100,
     "correct": 57,
     "accuracy": 0.57,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 15.085872479947284,
     "p95_request_seconds": 74.065677969018,
     "input_tokens": 23548,
     "output_tokens": 113588,
     "cost_usd": 0.14337,
     "cost_per_million_questions": 1433.7,
     "served_models": [
      "deepseek-v4.1-flash-1017"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9989019175991835,
      "sure": 97,
      "sure_wrong": 42
     }
    }
   ],
   "accuracy": {
    "mean": 0.5933333333333333,
    "min": 0.57,
    "max": 0.63
   },
   "accuracy_ci95": {
    "low": 0.51,
    "high": 0.6733333333333333
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 16.47434018664838,
    "min": 15.085872479947284,
    "max": 18.741823738033418
   },
   "p95_request_seconds": {
    "mean": 70.1002825757023,
    "min": 64.2766059130663,
    "max": 74.065677969018
   },
   "cost_per_million_questions": {
    "mean": 1368.032,
    "min": 1288.332,
    "max": 1433.7
   },
   "host": "our GPU",
   "run": 585,
   "reasoning": "on",
   "access": "SGLang, self-hosted, published mixed precision; 1017 weights (revision 1bae0e1)",
   "interface": null,
   "billing": "local",
   "workers": 4,
   "best_setting": true,
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.999134571709431,
    "min": 0.9987798592538969,
    "max": 0.9997219382752126
   },
   "sure_wrong": {
    "mean": 40.333333333333336,
    "min": 37,
    "max": 42
   },
   "sure": {
    "mean": 98.66666666666667,
    "min": 97,
    "max": 100
   },
   "cost_kind": "API-equivalent estimate",
   "cost_note": null,
   "method_note": null
  },
  {
   "provider": "vllm",
   "model": "deepseek-v4.1-flash-1017",
   "perplexity_mode": null,
   "request_model": "deepseek-v4.1-flash-1017",
   "group": "chat",
   "web": false,
   "price": {
    "basis": "DeepSeek's list price for the same model",
    "source": "https://api-docs.deepseek.com/quick_start/pricing",
    "date": "2026-11-05",
    "input_per_million": 0.3,
    "per_request": 0.0,
    "output_per_million": 1.2,
    "applies_to": "maker",
    "per_search": 0.0
   },
   "runs": [
    {
     "started": "2026-11-05T22:38:28+00:00",
     "ended": "2026-11-05T22:39:02+00:00",
     "questions": 100,
     "correct": 43,
     "accuracy": 0.43,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 1.3391794560011476,
     "p95_request_seconds": 1.9018267400097102,
     "input_tokens": 21048,
     "output_tokens": 1198,
     "cost_usd": 0.007751999999999999,
     "cost_per_million_questions": 77.52,
     "served_models": [
      "deepseek-v4.1-flash-1017"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7825622311122692,
      "sure": 7,
      "sure_wrong": 3
     }
    },
    {
     "started": "2026-11-05T22:39:03+00:00",
     "ended": "2026-11-05T22:39:21+00:00",
     "questions": 100,
     "correct": 43,
     "accuracy": 0.43,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.7353282429976389,
     "p95_request_seconds": 0.9614140560152009,
     "input_tokens": 21048,
     "output_tokens": 1200,
     "cost_usd": 0.0077544,
     "cost_per_million_questions": 77.544,
     "served_models": [
      "deepseek-v4.1-flash-1017"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7822242802271628,
      "sure": 8,
      "sure_wrong": 3
     }
    },
    {
     "started": "2026-11-05T22:39:21+00:00",
     "ended": "2026-11-05T22:39:40+00:00",
     "questions": 100,
     "correct": 45,
     "accuracy": 0.45,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 0.7575958344968967,
     "p95_request_seconds": 0.9463316969340667,
     "input_tokens": 21048,
     "output_tokens": 1198,
     "cost_usd": 0.007751999999999999,
     "cost_per_million_questions": 77.52,
     "served_models": [
      "deepseek-v4.1-flash-1017"
     ],
     "re_asked": 0,
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.7824386489469506,
      "sure": 6,
      "sure_wrong": 2
     }
    }
   ],
   "accuracy": {
    "mean": 0.43666666666666665,
    "min": 0.43,
    "max": 0.45
   },
   "accuracy_ci95": {
    "low": 0.3433333333333334,
    "high": 0.5333333333333333
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 0.9440345111652277,
    "min": 0.7353282429976389,
    "max": 1.3391794560011476
   },
   "p95_request_seconds": {
    "mean": 1.2698574976529926,
    "min": 0.9463316969340667,
    "max": 1.9018267400097102
   },
   "cost_per_million_questions": {
    "mean": 77.52799999999999,
    "min": 77.52,
    "max": 77.544
   },
   "host": "our GPU",
   "run": 558,
   "reasoning": "off",
   "access": "SGLang, self-hosted, published mixed precision; 1017 weights (revision 1bae0e1)",
   "interface": null,
   "billing": "local",
   "workers": 4,
   "best_setting": false,
   "confidence_kind": "probability",
   "mean_chosen_probability": {
    "mean": 0.7824083867621275,
    "min": 0.7822242802271628,
    "max": 0.7825622311122692
   },
   "sure_wrong": {
    "mean": 2.6666666666666665,
    "min": 2,
    "max": 3
   },
   "sure": {
    "mean": 7,
    "min": 6,
    "max": 8
   },
   "cost_kind": "API-equivalent estimate",
   "cost_note": null,
   "method_note": null
  },
  {
   "provider": "openai",
   "model": "gpt-6-astra",
   "perplexity_mode": null,
   "request_model": "gpt-6-astra",
   "group": "chat",
   "web": true,
   "price": {
    "basis": "OpenAI's API list price",
    "source": "https://developers.openai.com/api/docs/pricing",
    "date": "2026-10-08",
    "input_per_million": 10.0,
    "per_request": 0.0,
    "output_per_million": 50.0,
    "applies_to": "billed",
    "per_search": 0.01,
    "searches": "counted",
    "search_source": "https://developers.openai.com/api/docs/pricing",
    "search_note": "$10 per 1,000 web search calls; search content tokens billed at the model's rates"
   },
   "runs": [
    {
     "started": "2026-11-05T23:31:03+00:00",
     "ended": "2026-11-05T23:40:58+00:00",
     "questions": 100,
     "correct": 100,
     "accuracy": 1.0,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 9.38164432294434,
     "p95_request_seconds": 23.130375129985623,
     "input_tokens": 3402423,
     "output_tokens": 12012,
     "cost_usd": 18.899534,
     "cost_per_million_questions": 188995.34,
     "served_models": [],
     "re_asked": 0,
     "searches": {
      "reported": true,
      "total": 161,
      "per_question": 1.61,
      "questions_with_search": 100
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.99275,
      "sure": 94,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T23:41:06+00:00",
     "ended": "2026-11-05T23:50:39+00:00",
     "questions": 100,
     "correct": 99,
     "accuracy": 0.99,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 9.349891752470285,
     "p95_request_seconds": 21.83097423997242,
     "input_tokens": 3215830,
     "output_tokens": 11485,
     "cost_usd": 17.264374,
     "cost_per_million_questions": 172643.74,
     "served_models": [],
     "re_asked": 0,
     "searches": {
      "reported": true,
      "total": 151,
      "per_question": 1.51,
      "questions_with_search": 100
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.99178,
      "sure": 91,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T23:50:47+00:00",
     "ended": "2026-11-06T00:00:37+00:00",
     "questions": 100,
     "correct": 100,
     "accuracy": 1.0,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 9.713032957457472,
     "p95_request_seconds": 22.68322641996201,
     "input_tokens": 3331114,
     "output_tokens": 11800,
     "cost_usd": 17.085588,
     "cost_per_million_questions": 170855.88,
     "served_models": [],
     "re_asked": 0,
     "searches": {
      "reported": true,
      "total": 156,
      "per_question": 1.56,
      "questions_with_search": 100
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9905,
      "sure": 90,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.9966666666666667,
    "min": 0.99,
    "max": 1.0
   },
   "accuracy_ci95": {
    "low": 0.99,
    "high": 1.0
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 9.481523010957366,
    "min": 9.349891752470285,
    "max": 9.713032957457472
   },
   "p95_request_seconds": {
    "mean": 22.548191929973353,
    "min": 21.83097423997242,
    "max": 23.130375129985623
   },
   "cost_per_million_questions": {
    "mean": 177498.32,
    "min": 170855.88,
    "max": 188995.34
   },
   "run": 277,
   "reasoning": "high",
   "access": "Codex CLI, ChatGPT subscription",
   "interface": "CLI",
   "billing": "subscription",
   "workers": 2,
   "best_setting": true,
   "confidence_kind": "stated",
   "cost_lower_bound": false,
   "searches_per_question": {
    "mean": 1.56,
    "min": 1.51,
    "max": 1.61
   },
   "questions_with_search": {
    "mean": 100,
    "min": 100,
    "max": 100
   },
   "mean_chosen_probability": {
    "mean": 0.9916766666666667,
    "min": 0.9905,
    "max": 0.99275
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 91.66666666666667,
    "min": 90,
    "max": 94
   },
   "cost_kind": "API-equivalent estimate",
   "cost_note": null,
   "method_note": "the Codex CLI adds its own context items to every request: 4,278 to 4,284 more input tokens than OpenAI's Responses API took for the same question (257 there; 62 questions compared), counted in the cost"
  },
  {
   "provider": "openai",
   "model": "gpt-6.1-sol",
   "perplexity_mode": null,
   "request_model": "gpt-6.1-sol",
   "group": "chat",
   "web": true,
   "price": {
    "basis": "OpenAI's API list price",
    "source": "https://developers.openai.com/api/docs/pricing",
    "date": "2026-10-08",
    "input_per_million": 2.0,
    "per_request": 0.0,
    "output_per_million": 10.0,
    "applies_to": "billed",
    "per_search": 0.01,
    "searches": "counted",
    "search_source": "https://developers.openai.com/api/docs/pricing",
    "search_note": "$10 per 1,000 web search calls; search content tokens billed at the model's rates"
   },
   "runs": [
    {
     "started": "2026-11-05T16:35:33+00:00",
     "ended": "2026-11-05T16:49:12+00:00",
     "questions": 100,
     "correct": 100,
     "accuracy": 1.0,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 12.829665725526866,
     "p95_request_seconds": 32.6908755491022,
     "input_tokens": 3909970,
     "output_tokens": 16408,
     "cost_usd": 5.924292,
     "cost_per_million_questions": 59242.920000000006,
     "served_models": [],
     "re_asked": 0,
     "searches": {
      "reported": true,
      "total": 175,
      "per_question": 1.75,
      "questions_with_search": 100
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.99284,
      "sure": 92,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:49:25+00:00",
     "ended": "2026-11-05T17:01:38+00:00",
     "questions": 100,
     "correct": 100,
     "accuracy": 1.0,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 12.411790462501813,
     "p95_request_seconds": 28.351989193004556,
     "input_tokens": 3603077,
     "output_tokens": 15723,
     "cost_usd": 5.2560720000000005,
     "cost_per_million_questions": 52560.72000000001,
     "served_models": [],
     "re_asked": 0,
     "searches": {
      "reported": true,
      "total": 161,
      "per_question": 1.61,
      "questions_with_search": 100
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.99411,
      "sure": 95,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T17:01:48+00:00",
     "ended": "2026-11-05T17:14:30+00:00",
     "questions": 100,
     "correct": 100,
     "accuracy": 1.0,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 13.33951168495696,
     "p95_request_seconds": 28.092054730979726,
     "input_tokens": 4020063,
     "output_tokens": 16315,
     "cost_usd": 5.5263352,
     "cost_per_million_questions": 55263.352,
     "served_models": [],
     "re_asked": 0,
     "searches": {
      "reported": true,
      "total": 176,
      "per_question": 1.76,
      "questions_with_search": 100
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.99397,
      "sure": 95,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 1.0,
    "min": 1.0,
    "max": 1.0
   },
   "accuracy_ci95": {
    "low": 1.0,
    "high": 1.0
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 12.860322624328546,
    "min": 12.411790462501813,
    "max": 13.33951168495696
   },
   "p95_request_seconds": {
    "mean": 29.711639824362162,
    "min": 28.092054730979726,
    "max": 32.6908755491022
   },
   "cost_per_million_questions": {
    "mean": 55688.99733333334,
    "min": 52560.72000000001,
    "max": 59242.920000000006
   },
   "run": 276,
   "reasoning": "high",
   "access": "Codex CLI, ChatGPT subscription",
   "interface": "CLI",
   "billing": "subscription",
   "workers": 2,
   "best_setting": true,
   "confidence_kind": "stated",
   "cost_lower_bound": false,
   "searches_per_question": {
    "mean": 1.7066666666666668,
    "min": 1.61,
    "max": 1.76
   },
   "questions_with_search": {
    "mean": 100,
    "min": 100,
    "max": 100
   },
   "mean_chosen_probability": {
    "mean": 0.99364,
    "min": 0.99284,
    "max": 0.99411
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 94,
    "min": 92,
    "max": 95
   },
   "cost_kind": "API-equivalent estimate",
   "cost_note": null,
   "method_note": "the Codex CLI adds its own context items to every request: 4,278 to 4,284 more input tokens than OpenAI's Responses API took for the same question (257 there; 62 questions compared), counted in the cost"
  },
  {
   "provider": "openai",
   "model": "gpt-6.1-terra",
   "perplexity_mode": null,
   "request_model": "gpt-6.1-terra",
   "group": "chat",
   "web": true,
   "price": {
    "basis": "OpenAI's API list price",
    "source": "https://developers.openai.com/api/docs/pricing",
    "date": "2026-10-28",
    "input_per_million": 1.0,
    "per_request": 0.0,
    "output_per_million": 5.0,
    "applies_to": "billed",
    "per_search": 0.01,
    "searches": "counted",
    "search_source": "https://developers.openai.com/api/docs/pricing",
    "search_note": "$10 per 1,000 web search calls; search content tokens billed at the model's rates"
   },
   "runs": [
    {
     "started": "2026-11-05T17:14:45+00:00",
     "ended": "2026-11-05T17:23:47+00:00",
     "questions": 100,
     "correct": 99,
     "accuracy": 0.99,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 9.5322861779714,
     "p95_request_seconds": 18.559949612012133,
     "input_tokens": 2462301,
     "output_tokens": 12437,
     "cost_usd": 2.429446,
     "cost_per_million_questions": 24294.46,
     "served_models": [],
     "re_asked": 0,
     "searches": {
      "reported": true,
      "total": 120,
      "per_question": 1.2,
      "questions_with_search": 100
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.99135,
      "sure": 93,
      "sure_wrong": 1
     }
    },
    {
     "started": "2026-11-05T17:23:56+00:00",
     "ended": "2026-11-05T17:46:31+00:00",
     "questions": 100,
     "correct": 99,
     "accuracy": 0.99,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 9.666686348500662,
     "p95_request_seconds": 18.292146848980337,
     "input_tokens": 2480421,
     "output_tokens": 12286,
     "cost_usd": 2.3906318,
     "cost_per_million_questions": 23906.318,
     "served_models": [],
     "re_asked": 1,
     "searches": {
      "reported": true,
      "total": 120,
      "per_question": 1.2,
      "questions_with_search": 100
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.99194,
      "sure": 95,
      "sure_wrong": 1
     }
    },
    {
     "started": "2026-11-05T17:33:10+00:00",
     "ended": "2026-11-05T17:42:09+00:00",
     "questions": 100,
     "correct": 99,
     "accuracy": 0.99,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 9.983510723977815,
     "p95_request_seconds": 16.386625156039372,
     "input_tokens": 2473257,
     "output_tokens": 12620,
     "cost_usd": 2.401461,
     "cost_per_million_questions": 24014.609999999997,
     "served_models": [],
     "re_asked": 0,
     "searches": {
      "reported": true,
      "total": 121,
      "per_question": 1.21,
      "questions_with_search": 100
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.99126,
      "sure": 94,
      "sure_wrong": 1
     }
    }
   ],
   "accuracy": {
    "mean": 0.99,
    "min": 0.99,
    "max": 0.99
   },
   "accuracy_ci95": {
    "low": 0.97,
    "high": 1.0
   },
   "re_asked": [
    0,
    1,
    0
   ],
   "median_request_seconds": {
    "mean": 9.727494416816626,
    "min": 9.5322861779714,
    "max": 9.983510723977815
   },
   "p95_request_seconds": {
    "mean": 17.746240539010614,
    "min": 16.386625156039372,
    "max": 18.559949612012133
   },
   "cost_per_million_questions": {
    "mean": 24071.796,
    "min": 23906.318,
    "max": 24294.46
   },
   "run": 275,
   "reasoning": "high",
   "access": "Codex CLI, ChatGPT subscription",
   "interface": "CLI",
   "billing": "subscription",
   "workers": 2,
   "best_setting": true,
   "confidence_kind": "stated",
   "cost_lower_bound": false,
   "searches_per_question": {
    "mean": 1.2033333333333334,
    "min": 1.2,
    "max": 1.21
   },
   "questions_with_search": {
    "mean": 100,
    "min": 100,
    "max": 100
   },
   "mean_chosen_probability": {
    "mean": 0.9915166666666667,
    "min": 0.99126,
    "max": 0.99194
   },
   "sure_wrong": {
    "mean": 1,
    "min": 1,
    "max": 1
   },
   "sure": {
    "mean": 94,
    "min": 93,
    "max": 95
   },
   "cost_kind": "API-equivalent estimate",
   "cost_note": null,
   "method_note": "the Codex CLI adds its own context items to every request: 4,278 to 4,284 more input tokens than OpenAI's Responses API took for the same question (257 there; 62 questions compared), counted in the cost"
  },
  {
   "provider": "openai",
   "model": "gpt-6-luna",
   "perplexity_mode": null,
   "request_model": "gpt-6-luna",
   "group": "chat",
   "web": true,
   "price": {
    "basis": "OpenAI's API list price",
    "source": "https://developers.openai.com/api/docs/pricing",
    "date": "2026-10-08",
    "input_per_million": 0.1,
    "per_request": 0.0,
    "output_per_million": 0.5,
    "applies_to": "billed",
    "per_search": 0.01,
    "searches": "counted",
    "search_source": "https://developers.openai.com/api/docs/pricing",
    "search_note": "$10 per 1,000 web search calls; search content tokens billed at the model's rates"
   },
   "runs": [
    {
     "started": "2026-11-05T17:42:24+00:00",
     "ended": "2026-11-05T17:47:43+00:00",
     "questions": 100,
     "correct": 67,
     "accuracy": 0.67,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 4.9114736299961805,
     "p95_request_seconds": 11.753761084051803,
     "input_tokens": 1263402,
     "output_tokens": 5581,
     "cost_usd": 0.54782062,
     "cost_per_million_questions": 5478.2062,
     "served_models": [],
     "re_asked": 0,
     "searches": {
      "reported": true,
      "total": 47,
      "per_question": 0.47,
      "questions_with_search": 42
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9443,
      "sure": 49,
      "sure_wrong": 7
     }
    },
    {
     "started": "2026-11-05T17:47:46+00:00",
     "ended": "2026-11-05T17:53:14+00:00",
     "questions": 100,
     "correct": 63,
     "accuracy": 0.63,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 4.4044490164960735,
     "p95_request_seconds": 12.475402100011706,
     "input_tokens": 1188507,
     "output_tokens": 5134,
     "cost_usd": 0.47282698,
     "cost_per_million_questions": 4728.269800000001,
     "served_models": [],
     "re_asked": 0,
     "searches": {
      "reported": true,
      "total": 41,
      "per_question": 0.41,
      "questions_with_search": 33
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9308,
      "sure": 40,
      "sure_wrong": 6
     }
    },
    {
     "started": "2026-11-05T17:53:18+00:00",
     "ended": "2026-11-05T18:00:09+00:00",
     "questions": 100,
     "correct": 61,
     "accuracy": 0.61,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 8.398811651510186,
     "p95_request_seconds": 15.71514747000765,
     "input_tokens": 1151736,
     "output_tokens": 5141,
     "cost_usd": 0.44520202000000003,
     "cost_per_million_questions": 4452.0202,
     "served_models": [],
     "re_asked": 0,
     "searches": {
      "reported": true,
      "total": 39,
      "per_question": 0.39,
      "questions_with_search": 34
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9396,
      "sure": 38,
      "sure_wrong": 8
     }
    }
   ],
   "accuracy": {
    "mean": 0.6366666666666667,
    "min": 0.61,
    "max": 0.67
   },
   "accuracy_ci95": {
    "low": 0.5566666666666666,
    "high": 0.7133333333333333
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 5.90491143266748,
    "min": 4.4044490164960735,
    "max": 8.398811651510186
   },
   "p95_request_seconds": {
    "mean": 13.31477021802372,
    "min": 11.753761084051803,
    "max": 15.71514747000765
   },
   "cost_per_million_questions": {
    "mean": 4886.1654,
    "min": 4452.0202,
    "max": 5478.2062
   },
   "run": 133,
   "reasoning": "low",
   "access": "Codex CLI, ChatGPT subscription",
   "interface": "CLI",
   "billing": "subscription",
   "workers": 2,
   "best_setting": true,
   "confidence_kind": "stated",
   "cost_lower_bound": false,
   "searches_per_question": {
    "mean": 0.42333333333333334,
    "min": 0.39,
    "max": 0.47
   },
   "questions_with_search": {
    "mean": 36.333333333333336,
    "min": 33,
    "max": 42
   },
   "mean_chosen_probability": {
    "mean": 0.9382333333333334,
    "min": 0.9308,
    "max": 0.9443
   },
   "sure_wrong": {
    "mean": 7,
    "min": 6,
    "max": 8
   },
   "sure": {
    "mean": 42.333333333333336,
    "min": 38,
    "max": 49
   },
   "cost_kind": "API-equivalent estimate",
   "cost_note": null,
   "method_note": "the Codex CLI adds its own context items to every request: 4,278 to 4,284 more input tokens than OpenAI's Responses API took for the same question (257 there; 62 questions compared), counted in the cost"
  },
  {
   "provider": "gemini",
   "model": "gemini-3.8-flash",
   "perplexity_mode": null,
   "request_model": "gemini-3.8-flash",
   "group": "chat",
   "web": true,
   "price": {
    "basis": "Google's Gemini API price, introductory",
    "source": "https://ai.google.dev/gemini-api/docs/pricing",
    "date": "2026-10-08",
    "input_per_million": 0.75,
    "per_request": 0.0,
    "output_per_million": 3.75,
    "applies_to": "billed",
    "per_search": 0.014,
    "searches": "not reported",
    "search_source": "https://ai.google.dev/gemini-api/docs/google-search",
    "search_note": "Gemini 3 and newer: $14 per 1,000 search queries the model runs, after 5,000 free a month (not deducted here)"
   },
   "runs": [
    {
     "started": "2026-11-05T16:24:10+00:00",
     "ended": "2026-11-05T16:31:24+00:00",
     "questions": 100,
     "correct": 98,
     "accuracy": 0.98,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 11.293122344475705,
     "p95_request_seconds": 38.792138371034525,
     "input_tokens": 84245,
     "output_tokens": 109703,
     "cost_usd": 0.45120555,
     "cost_per_million_questions": 4512.0554999999995,
     "served_models": [
      "gemini-3.8-flash"
     ],
     "re_asked": 0,
     "searches": {
      "reported": false,
      "total": null,
      "per_question": null,
      "questions_with_search": null
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9924,
      "sure": 84,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:31:45+00:00",
     "ended": "2026-11-05T16:39:14+00:00",
     "questions": 100,
     "correct": 98,
     "accuracy": 0.98,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 11.625887635455001,
     "p95_request_seconds": 45.708553300006315,
     "input_tokens": 94028,
     "output_tokens": 125034,
     "cost_usd": 0.511328625,
     "cost_per_million_questions": 5113.28625,
     "served_models": [
      "gemini-3.8-flash"
     ],
     "re_asked": 0,
     "searches": {
      "reported": false,
      "total": null,
      "per_question": null,
      "questions_with_search": null
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9919,
      "sure": 83,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:39:29+00:00",
     "ended": "2026-11-05T16:46:55+00:00",
     "questions": 100,
     "correct": 99,
     "accuracy": 0.99,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 11.925346768985037,
     "p95_request_seconds": 55.20039250492118,
     "input_tokens": 82551,
     "output_tokens": 115310,
     "cost_usd": 0.471731475,
     "cost_per_million_questions": 4717.31475,
     "served_models": [
      "gemini-3.8-flash"
     ],
     "re_asked": 0,
     "searches": {
      "reported": false,
      "total": null,
      "per_question": null,
      "questions_with_search": null
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9901,
      "sure": 78,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.9833333333333333,
    "min": 0.98,
    "max": 0.99
   },
   "accuracy_ci95": {
    "low": 0.9566666666666667,
    "high": 1.0
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 11.614785582971914,
    "min": 11.293122344475705,
    "max": 11.925346768985037
   },
   "p95_request_seconds": {
    "mean": 46.56702805865401,
    "min": 38.792138371034525,
    "max": 55.20039250492118
   },
   "cost_per_million_questions": {
    "mean": 4780.885499999999,
    "min": 4512.0554999999995,
    "max": 5113.28625
   },
   "run": 140,
   "reasoning": "high",
   "access": "Gemini API",
   "interface": null,
   "billing": "api",
   "workers": 4,
   "best_setting": true,
   "confidence_kind": "stated",
   "cost_lower_bound": true,
   "searches_per_question": null,
   "questions_with_search": null,
   "mean_chosen_probability": {
    "mean": 0.9914666666666666,
    "min": 0.9901,
    "max": 0.9924
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 81.66666666666667,
    "min": 78,
    "max": 84
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": "the search fee is missing: the API reports no search count, so the estimate covers the tokens only",
   "method_note": null
  },
  {
   "provider": "gemini",
   "model": "gemini-4-argon",
   "perplexity_mode": null,
   "request_model": "gemini-4-argon",
   "group": "chat",
   "web": true,
   "price": {
    "basis": "Google's Gemini API regular price (from 1 December 2026)",
    "source": "https://ai.google.dev/gemini-api/docs/pricing",
    "date": "2026-10-28",
    "input_per_million": 4.0,
    "per_request": 0.0,
    "output_per_million": 20.0,
    "applies_to": "billed",
    "per_search": 0.014,
    "searches": "not reported",
    "search_source": "https://ai.google.dev/gemini-api/docs/google-search",
    "search_note": "Gemini 3 and newer: $14 per 1,000 search queries the model runs, after 5,000 free a month (not deducted here)"
   },
   "runs": [
    {
     "started": "2026-11-05T16:47:08+00:00",
     "ended": "2026-11-05T16:57:16+00:00",
     "questions": 100,
     "correct": 100,
     "accuracy": 1.0,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 12.607490956492256,
     "p95_request_seconds": 87.45263422606513,
     "input_tokens": 71771,
     "output_tokens": 14637,
     "cost_usd": 0.579824,
     "cost_per_million_questions": 5798.24,
     "served_models": [
      "gemini-4-argon"
     ],
     "re_asked": 0,
     "searches": {
      "reported": false,
      "total": null,
      "per_question": null,
      "questions_with_search": null
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9921,
      "sure": 90,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:57:37+00:00",
     "ended": "2026-11-05T17:10:59+00:00",
     "questions": 100,
     "correct": 100,
     "accuracy": 1.0,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 14.2144973075483,
     "p95_request_seconds": 112.34591604291927,
     "input_tokens": 71771,
     "output_tokens": 18288,
     "cost_usd": 0.652844,
     "cost_per_million_questions": 6528.44,
     "served_models": [
      "gemini-4-argon"
     ],
     "re_asked": 0,
     "searches": {
      "reported": false,
      "total": null,
      "per_question": null,
      "questions_with_search": null
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9906,
      "sure": 92,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T17:11:22+00:00",
     "ended": "2026-11-05T17:19:52+00:00",
     "questions": 100,
     "correct": 99,
     "accuracy": 0.99,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 9.82701177551644,
     "p95_request_seconds": 65.31190391699784,
     "input_tokens": 71771,
     "output_tokens": 15339,
     "cost_usd": 0.593864,
     "cost_per_million_questions": 5938.639999999999,
     "served_models": [
      "gemini-4-argon"
     ],
     "re_asked": 0,
     "searches": {
      "reported": false,
      "total": null,
      "per_question": null,
      "questions_with_search": null
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9919,
      "sure": 96,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.9966666666666667,
    "min": 0.99,
    "max": 1.0
   },
   "accuracy_ci95": {
    "low": 0.99,
    "high": 1.0
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 12.216333346518999,
    "min": 9.82701177551644,
    "max": 14.2144973075483
   },
   "p95_request_seconds": {
    "mean": 88.37015139532741,
    "min": 65.31190391699784,
    "max": 112.34591604291927
   },
   "cost_per_million_questions": {
    "mean": 6088.44,
    "min": 5798.24,
    "max": 6528.44
   },
   "run": 557,
   "reasoning": "low",
   "access": "Gemini API",
   "interface": null,
   "billing": "api",
   "workers": 4,
   "best_setting": true,
   "confidence_kind": "stated",
   "cost_lower_bound": true,
   "searches_per_question": null,
   "questions_with_search": null,
   "mean_chosen_probability": {
    "mean": 0.9915333333333334,
    "min": 0.9906,
    "max": 0.9921
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 92.66666666666667,
    "min": 90,
    "max": 96
   },
   "cost_kind": "API usage cost estimate",
   "cost_note": "the search fee is missing: the API reports no search count, so the estimate covers the tokens only",
   "method_note": null
  },
  {
   "provider": "claude",
   "model": "claude-fable-5-1",
   "perplexity_mode": null,
   "request_model": "claude-fable-5-1",
   "group": "chat",
   "web": true,
   "price": {
    "basis": "Anthropic's API list price",
    "source": "https://platform.claude.com/docs/en/about-claude/pricing",
    "date": "2026-10-28",
    "input_per_million": 10.0,
    "per_request": 0.0,
    "output_per_million": 50.0,
    "applies_to": "billed",
    "per_search": 0.01,
    "searches": "counted",
    "search_source": "https://platform.claude.com/docs/en/agents-and-tools/tool-use/web-search-tool",
    "search_note": "$10 per 1,000 searches plus the tokens of the search results"
   },
   "runs": [
    {
     "started": "2026-11-05T16:24:11+00:00",
     "ended": "2026-11-05T16:33:35+00:00",
     "questions": 100,
     "correct": 100,
     "accuracy": 1.0,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 18.512011685001198,
     "p95_request_seconds": 68.26213914901018,
     "input_tokens": 4058986,
     "output_tokens": 251714,
     "cost_usd": 14.0391549,
     "cost_per_million_questions": 140391.549,
     "served_models": [
      "claude-fable-5-1",
      "claude-haiku-5-5"
     ],
     "re_asked": 0,
     "stand_in_refusals": 0,
     "searches": {
      "reported": true,
      "total": 224,
      "per_question": 2.24,
      "questions_with_search": 94
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.8662,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:33:47+00:00",
     "ended": "2026-11-05T16:42:06+00:00",
     "questions": 100,
     "correct": 99,
     "accuracy": 0.99,
     "missing": 0,
     "not_success": 1,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 18.32121833844576,
     "p95_request_seconds": 45.28962390997913,
     "input_tokens": 3536600,
     "output_tokens": 219042,
     "cost_usd": 11.299611350000001,
     "cost_per_million_questions": 112996.1135,
     "served_models": [
      "claude-fable-5-1",
      "claude-haiku-5-5",
      "claude-opus-4-8"
     ],
     "re_asked": 0,
     "stand_in_refusals": 1,
     "searches": {
      "reported": true,
      "total": 193,
      "per_question": 1.93,
      "questions_with_search": 90
     },
     "confidence": {
      "answers": 99,
      "mean_chosen_probability": 0.8715151515151515,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:42:21+00:00",
     "ended": "2026-11-05T16:50:36+00:00",
     "questions": 100,
     "correct": 98,
     "accuracy": 0.98,
     "missing": 0,
     "not_success": 1,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 15.932620437059086,
     "p95_request_seconds": 44.185549605987035,
     "input_tokens": 3386893,
     "output_tokens": 215664,
     "cost_usd": 10.96608485,
     "cost_per_million_questions": 109660.8485,
     "served_models": [
      "claude-fable-5-1",
      "claude-haiku-5-5",
      "claude-opus-4-8"
     ],
     "re_asked": 0,
     "stand_in_refusals": 1,
     "searches": {
      "reported": true,
      "total": 185,
      "per_question": 1.85,
      "questions_with_search": 91
     },
     "confidence": {
      "answers": 99,
      "mean_chosen_probability": 0.8702020202020202,
      "sure": 1,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.99,
    "min": 0.98,
    "max": 1.0
   },
   "accuracy_ci95": {
    "low": 0.9733333333333333,
    "high": 1.0
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 17.58861682016868,
    "min": 15.932620437059086,
    "max": 18.512011685001198
   },
   "p95_request_seconds": {
    "mean": 52.57910422165878,
    "min": 44.185549605987035,
    "max": 68.26213914901018
   },
   "cost_per_million_questions": {
    "mean": 121016.17033333333,
    "min": 109660.8485,
    "max": 140391.549
   },
   "run": 310,
   "reasoning": "high",
   "access": "Claude Code CLI, Claude subscription",
   "interface": "CLI",
   "billing": "subscription",
   "workers": 4,
   "best_setting": true,
   "confidence_kind": "stated",
   "cost_lower_bound": false,
   "searches_per_question": {
    "mean": 2.006666666666667,
    "min": 1.85,
    "max": 2.24
   },
   "questions_with_search": {
    "mean": 91.66666666666667,
    "min": 90,
    "max": 94
   },
   "mean_chosen_probability": {
    "mean": 0.8693057239057239,
    "min": 0.8662,
    "max": 0.8715151515151515
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0.3333333333333333,
    "min": 0,
    "max": 1
   },
   "cost_kind": "API-equivalent estimate",
   "cost_note": null,
   "method_note": "Claude Code writes the prompt to its 1-hour cache, counted in the cost; Claude Code runs each web search through a separate Claude Haiku 5.5 call, whose tokens are counted in the cost at Haiku 5.5's list price",
   "stand_in": {
    "models": [
     "claude-opus-4-8"
    ],
    "requests": [
     0,
     1,
     1
    ],
    "questions": 1,
    "scored": "as failures (refusals), not with the other model's answers"
   }
  },
  {
   "provider": "claude",
   "model": "claude-opus-5-5",
   "perplexity_mode": null,
   "request_model": "claude-opus-5-5",
   "group": "chat",
   "web": true,
   "price": {
    "basis": "Anthropic's API list price",
    "source": "https://platform.claude.com/docs/en/about-claude/pricing",
    "date": "2026-10-08",
    "input_per_million": 4.0,
    "per_request": 0.0,
    "output_per_million": 20.0,
    "applies_to": "billed",
    "per_search": 0.01,
    "searches": "counted",
    "search_source": "https://platform.claude.com/docs/en/agents-and-tools/tool-use/web-search-tool",
    "search_note": "$10 per 1,000 searches plus the tokens of the search results"
   },
   "runs": [
    {
     "started": "2026-11-05T16:50:48+00:00",
     "ended": "2026-11-05T16:58:27+00:00",
     "questions": 100,
     "correct": 98,
     "accuracy": 0.98,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 17.027329092961736,
     "p95_request_seconds": 37.15850880497601,
     "input_tokens": 3054378,
     "output_tokens": 191610,
     "cost_usd": 5.5917777,
     "cost_per_million_questions": 55917.776999999995,
     "served_models": [
      "claude-haiku-5-5",
      "claude-opus-5-5"
     ],
     "re_asked": 0,
     "stand_in_refusals": 0,
     "searches": {
      "reported": true,
      "total": 162,
      "per_question": 1.62,
      "questions_with_search": 86
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.84,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:58:36+00:00",
     "ended": "2026-11-05T17:06:30+00:00",
     "questions": 100,
     "correct": 97,
     "accuracy": 0.97,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 16.258004982490093,
     "p95_request_seconds": 40.75289816991426,
     "input_tokens": 3013809,
     "output_tokens": 187649,
     "cost_usd": 5.0029759,
     "cost_per_million_questions": 50029.759,
     "served_models": [
      "claude-haiku-5-5",
      "claude-opus-5-5"
     ],
     "re_asked": 0,
     "stand_in_refusals": 0,
     "searches": {
      "reported": true,
      "total": 159,
      "per_question": 1.59,
      "questions_with_search": 84
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.8449,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T17:06:37+00:00",
     "ended": "2026-11-05T17:13:42+00:00",
     "questions": 100,
     "correct": 99,
     "accuracy": 0.99,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 15.408668294956442,
     "p95_request_seconds": 38.86386767192744,
     "input_tokens": 2744006,
     "output_tokens": 175853,
     "cost_usd": 4.646801,
     "cost_per_million_questions": 46468.009999999995,
     "served_models": [
      "claude-haiku-5-5",
      "claude-opus-5-5"
     ],
     "re_asked": 0,
     "stand_in_refusals": 0,
     "searches": {
      "reported": true,
      "total": 148,
      "per_question": 1.48,
      "questions_with_search": 85
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.8543,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.98,
    "min": 0.97,
    "max": 0.99
   },
   "accuracy_ci95": {
    "low": 0.9566666666666667,
    "high": 0.9966666666666667
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 16.231334123469424,
    "min": 15.408668294956442,
    "max": 17.027329092961736
   },
   "p95_request_seconds": {
    "mean": 38.925091548939236,
    "min": 37.15850880497601,
    "max": 40.75289816991426
   },
   "cost_per_million_questions": {
    "mean": 50805.18199999999,
    "min": 46468.009999999995,
    "max": 55917.776999999995
   },
   "run": 278,
   "reasoning": "high",
   "access": "Claude Code CLI, Claude subscription",
   "interface": "CLI",
   "billing": "subscription",
   "workers": 4,
   "best_setting": true,
   "confidence_kind": "stated",
   "cost_lower_bound": false,
   "searches_per_question": {
    "mean": 1.5633333333333335,
    "min": 1.48,
    "max": 1.62
   },
   "questions_with_search": {
    "mean": 85,
    "min": 84,
    "max": 86
   },
   "mean_chosen_probability": {
    "mean": 0.8463999999999999,
    "min": 0.84,
    "max": 0.8543
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "API-equivalent estimate",
   "cost_note": null,
   "method_note": "Claude Code writes the prompt to its 1-hour cache, counted in the cost; Claude Code runs each web search through a separate Claude Haiku 5.5 call, whose tokens are counted in the cost at Haiku 5.5's list price"
  },
  {
   "provider": "claude",
   "model": "claude-sonnet-5-5",
   "perplexity_mode": null,
   "request_model": "claude-sonnet-5-5",
   "group": "chat",
   "web": true,
   "price": {
    "basis": "Anthropic's API list price",
    "source": "https://platform.claude.com/docs/en/about-claude/pricing",
    "date": "2026-10-08",
    "input_per_million": 2.0,
    "per_request": 0.0,
    "output_per_million": 10.0,
    "applies_to": "billed",
    "per_search": 0.01,
    "searches": "counted",
    "search_source": "https://platform.claude.com/docs/en/agents-and-tools/tool-use/web-search-tool",
    "search_note": "$10 per 1,000 searches plus the tokens of the search results"
   },
   "runs": [
    {
     "started": "2026-11-05T17:13:53+00:00",
     "ended": "2026-11-05T17:20:01+00:00",
     "questions": 100,
     "correct": 95,
     "accuracy": 0.95,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 14.026071546017192,
     "p95_request_seconds": 28.917177046998404,
     "input_tokens": 2668374,
     "output_tokens": 163802,
     "cost_usd": 3.2451831,
     "cost_per_million_questions": 32451.831000000002,
     "served_models": [
      "claude-haiku-5-5",
      "claude-sonnet-5-5"
     ],
     "re_asked": 0,
     "stand_in_refusals": 0,
     "searches": {
      "reported": true,
      "total": 140,
      "per_question": 1.4,
      "questions_with_search": 73
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.8452,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T17:20:15+00:00",
     "ended": "2026-11-05T17:27:15+00:00",
     "questions": 100,
     "correct": 96,
     "accuracy": 0.96,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 14.113826596469153,
     "p95_request_seconds": 38.923382906010374,
     "input_tokens": 2946436,
     "output_tokens": 180231,
     "cost_usd": 3.3132017,
     "cost_per_million_questions": 33132.017,
     "served_models": [
      "claude-haiku-5-5",
      "claude-sonnet-5-5"
     ],
     "re_asked": 0,
     "stand_in_refusals": 0,
     "searches": {
      "reported": true,
      "total": 153,
      "per_question": 1.53,
      "questions_with_search": 73
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.8449,
      "sure": 0,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T17:27:38+00:00",
     "ended": "2026-11-05T17:34:19+00:00",
     "questions": 100,
     "correct": 98,
     "accuracy": 0.98,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 13.704808996990323,
     "p95_request_seconds": 38.04876556096133,
     "input_tokens": 2828336,
     "output_tokens": 168833,
     "cost_usd": 3.1559314,
     "cost_per_million_questions": 31559.314,
     "served_models": [
      "claude-haiku-5-5",
      "claude-sonnet-5-5"
     ],
     "re_asked": 0,
     "stand_in_refusals": 0,
     "searches": {
      "reported": true,
      "total": 147,
      "per_question": 1.47,
      "questions_with_search": 69
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.8452,
      "sure": 0,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.9633333333333333,
    "min": 0.95,
    "max": 0.98
   },
   "accuracy_ci95": {
    "low": 0.9333333333333332,
    "high": 0.9866666666666667
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 13.948235713158889,
    "min": 13.704808996990323,
    "max": 14.113826596469153
   },
   "p95_request_seconds": {
    "mean": 35.29644183799004,
    "min": 28.917177046998404,
    "max": 38.923382906010374
   },
   "cost_per_million_questions": {
    "mean": 32381.054,
    "min": 31559.314,
    "max": 33132.017
   },
   "run": 193,
   "reasoning": "high",
   "access": "Claude Code CLI, Claude subscription",
   "interface": "CLI",
   "billing": "subscription",
   "workers": 4,
   "best_setting": true,
   "confidence_kind": "stated",
   "cost_lower_bound": false,
   "searches_per_question": {
    "mean": 1.4666666666666666,
    "min": 1.4,
    "max": 1.53
   },
   "questions_with_search": {
    "mean": 71.66666666666667,
    "min": 69,
    "max": 73
   },
   "mean_chosen_probability": {
    "mean": 0.8451,
    "min": 0.8449,
    "max": 0.8452
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "cost_kind": "API-equivalent estimate",
   "cost_note": null,
   "method_note": "Claude Code writes the prompt to its 1-hour cache, counted in the cost; Claude Code runs each web search through a separate Claude Haiku 5.5 call, whose tokens are counted in the cost at Haiku 5.5's list price"
  },
  {
   "provider": "xai",
   "model": "grok-4.7",
   "perplexity_mode": null,
   "request_model": "grok-4.7",
   "group": "chat",
   "web": true,
   "price": {
    "basis": "xAI's API list price",
    "source": "https://docs.x.ai/docs/models/grok-4.7",
    "date": "2026-10-28",
    "input_per_million": 2.0,
    "per_request": 0.0,
    "output_per_million": 6.0,
    "applies_to": "billed",
    "per_search": 0.005,
    "searches": "counted",
    "search_source": "https://docs.x.ai/developers/pricing",
    "search_note": "$5 per 1,000 web_search calls plus tokens"
   },
   "runs": [
    {
     "started": "2026-11-05T16:24:10+00:00",
     "ended": "2026-11-05T16:34:32+00:00",
     "questions": 100,
     "correct": 98,
     "accuracy": 0.98,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 18.246919706987683,
     "p95_request_seconds": 75.70926408295054,
     "input_tokens": 17997599,
     "output_tokens": 155326,
     "cost_usd": 22.21477,
     "cost_per_million_questions": 222147.7,
     "served_models": [
      "grok-4.7"
     ],
     "re_asked": 0,
     "searches": {
      "reported": true,
      "total": 912,
      "per_question": 9.12,
      "questions_with_search": 100
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9395,
      "sure": 2,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:34:55+00:00",
     "ended": "2026-11-05T16:44:42+00:00",
     "questions": 100,
     "correct": 97,
     "accuracy": 0.97,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 16.255370325467084,
     "p95_request_seconds": 70.85813712200616,
     "input_tokens": 15537494,
     "output_tokens": 142719,
     "cost_usd": 20.499541999999998,
     "cost_per_million_questions": 204995.41999999998,
     "served_models": [
      "grok-4.7"
     ],
     "re_asked": 0,
     "searches": {
      "reported": true,
      "total": 878,
      "per_question": 8.78,
      "questions_with_search": 100
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.941,
      "sure": 3,
      "sure_wrong": 0
     }
    },
    {
     "started": "2026-11-05T16:47:41+00:00",
     "ended": "2026-11-05T16:56:26+00:00",
     "questions": 100,
     "correct": 100,
     "accuracy": 1.0,
     "missing": 0,
     "not_success": 0,
     "errors": 0,
     "retries": 0,
     "median_request_seconds": 15.385405455948785,
     "p95_request_seconds": 57.0305532629136,
     "input_tokens": 16004060,
     "output_tokens": 146106,
     "cost_usd": 20.554652,
     "cost_per_million_questions": 205546.52000000002,
     "served_models": [
      "grok-4.7"
     ],
     "re_asked": 0,
     "searches": {
      "reported": true,
      "total": 857,
      "per_question": 8.57,
      "questions_with_search": 100
     },
     "confidence": {
      "answers": 100,
      "mean_chosen_probability": 0.9445,
      "sure": 1,
      "sure_wrong": 0
     }
    }
   ],
   "accuracy": {
    "mean": 0.9833333333333333,
    "min": 0.97,
    "max": 1.0
   },
   "accuracy_ci95": {
    "low": 0.96,
    "high": 1.0
   },
   "re_asked": [
    0,
    0,
    0
   ],
   "median_request_seconds": {
    "mean": 16.62923182946785,
    "min": 15.385405455948785,
    "max": 18.246919706987683
   },
   "p95_request_seconds": {
    "mean": 67.86598482262343,
    "min": 57.0305532629136,
    "max": 75.70926408295054
   },
   "cost_per_million_questions": {
    "mean": 210896.54666666666,
    "min": 204995.41999999998,
    "max": 222147.7
   },
   "run": 311,
   "reasoning": "high",
   "access": "xAI Responses API, SuperGrok subscription",
   "interface": null,
   "billing": "subscription",
   "workers": 4,
   "best_setting": true,
   "confidence_kind": "stated",
   "cost_lower_bound": false,
   "searches_per_question": {
    "mean": 8.823333333333332,
    "min": 8.57,
    "max": 9.12
   },
   "questions_with_search": {
    "mean": 100,
    "min": 100,
    "max": 100
   },
   "mean_chosen_probability": {
    "mean": 0.9416666666666667,
    "min": 0.9395,
    "max": 0.9445
   },
   "sure_wrong": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "sure": {
    "mean": 2,
    "min": 1,
    "max": 3
   },
   "cost_kind": "API-equivalent estimate",
   "cost_note": null,
   "method_note": null
  }
 ],
 "not_run": [
  {
   "provider": "perplexity",
   "model": "pplx-decider-v1-27b",
   "reason": "Perplexity's API now answers requests for v1 with v1.2. On five of these questions, each of its answers came within 0.007 of one of three v1.2 answers, while v1.2's own three answers differed by up to 0.085."
  },
  {
   "provider": "perplexity",
   "model": "pplx-decider-v1.1-27b",
   "reason": "Perplexity's API now answers requests for v1.1 with v1.2. On five of these questions, each of its answers matched one of three v1.2 answers exactly, while v1.2's own three answers differed by up to 0.085."
  },
  {
   "provider": "fastino",
   "model": "fastino/GLiDE",
   "reason": "Since 10 October Fastino's API answers with its glide_v2 engine, as it did again on 5 November, not the glide engine of the site's run."
  }
 ],
 "baseten_check": {
  "questions": 20,
  "date": "2026-11-05",
  "baselabs/elephant-1": {
   "largest_gap": 0.283,
   "same_choice": 20
  },
  "baselabs/mammoth-1": {
   "largest_gap": 0.082,
   "same_choice": 19
  }
 },
 "habits": {
  "questions": 100,
  "right_positions": [
   68,
   26,
   5,
   1
  ],
  "yes_no": {
   "yes": {
    "questions": 53,
    "right": 42
   },
   "true": {
    "questions": 7,
    "right": 6
   }
  },
  "positive_first": true,
  "two_choice_other": 17,
  "first_right_other": 20,
  "lazy": {
   "always_first": 0.68,
   "always_last": 0.19,
   "most_often": {
    "position": 1,
    "score": 0.68
   },
   "always_yes": {
    "right": 48,
    "questions": 60
   },
   "yes_then_uniform": 0.6266666666666666,
   "random": 0.44666666666666666
  },
  "explained_margin": 0.01,
  "hard": {
   "share": 0.2,
   "questions": 10,
   "by_choices": {
    "2": 5,
    "4": 5
   },
   "first_right": 7,
   "uniform_expected": 3.75
  },
  "first_share_accuracy_r": 0.9267251703801742,
  "weights": {
   "source": "Hugging Face API",
   "read": "2026-11-06"
  },
  "public_records": {
   "dataset_created": "2026-11-01T23:49Z",
   "dataset_commit": "64da0d8",
   "dataset_committed": "2026-11-02T01:37Z",
   "repository_commit": "0f403cf",
   "repository_committed": "2026-11-02T17:33Z"
  }
 },
 "pending": {
  "parallel": "Parallel's own Decisions API: no key on this machine yet.",
  "gpu": "Runs on our GPUs: waits for a GPU window.",
  "chat_gpu": [],
  "chat_unfinished": []
 },
 "blocked": [
  {
   "setup": "gpt-6.1-sol-api",
   "model": "gpt-6.1-sol",
   "web": false,
   "interface": "API",
   "reason": "waiting for OpenAI API credit (the first attempt failed when the credit ran out and is kept apart)"
  },
  {
   "setup": "claude-fable-5-1-api",
   "model": "claude-fable-5-1",
   "web": false,
   "interface": "API",
   "reason": "not run: no Anthropic API key on our machine"
  },
  {
   "setup": "claude-opus-5-5-api",
   "model": "claude-opus-5-5",
   "web": false,
   "interface": "API",
   "reason": "not run: no Anthropic API key on our machine"
  },
  {
   "setup": "claude-sonnet-5-5-api",
   "model": "claude-sonnet-5-5",
   "web": false,
   "interface": "API",
   "reason": "not run: no Anthropic API key on our machine"
  },
  {
   "setup": "gpt-6.1-sol-api-web",
   "model": "gpt-6.1-sol",
   "web": true,
   "interface": "API",
   "reason": "waiting for OpenAI API credit (the first attempt failed when the credit ran out and is kept apart)"
  },
  {
   "setup": "claude-fable-5-1-api-web",
   "model": "claude-fable-5-1",
   "web": true,
   "interface": "API",
   "reason": "not run: no Anthropic API key on our machine"
  },
  {
   "setup": "claude-opus-5-5-api-web",
   "model": "claude-opus-5-5",
   "web": true,
   "interface": "API",
   "reason": "not run: no Anthropic API key on our machine"
  },
  {
   "setup": "claude-sonnet-5-5-api-web",
   "model": "claude-sonnet-5-5",
   "web": true,
   "interface": "API",
   "reason": "not run: no Anthropic API key on our machine"
  }
 ],
 "chat": {
  "request": "PWS-Decisions has no chat request of its own, so each question goes out as the benchmark's request (the state, the question's instructions, its choices as lettered options) in the site's English chat prompts, one question per request; the reply is one letter and a 0-1 confidence in JSON, scored with PWS-Decisions' scorer.",
  "system_prompt": "You are taking an English multiple-choice benchmark. Select exactly one supplied option per question and state your confidence in each answer. Do not explain.",
  "instruction": "Answer each of the following multiple-choice questions. Without explanation, return for each question the letter of the correct option together with a confidence score between 0 and 1 showing how sure you are that your answer is correct, in the specified JSON format.",
  "timeout_seconds": 900.0,
  "setting": "each model's best Text tab setting (highest Turkish accuracy)",
  "web_search": {
   "openai": {
    "how": "Codex CLI with web_search=\"live\" (live web access; the default \"cached\" uses an OpenAI-kept index)",
    "doc": "https://developers.openai.com/codex/config-reference",
    "read": "2026-11-05",
    "price_source": "https://developers.openai.com/api/docs/pricing",
    "per_search": 0.01,
    "price_note": "$10 per 1,000 web search calls; search content tokens billed at the model's rates"
   },
   "claude": {
    "how": "Claude Code CLI with its WebSearch tool (the Claude API's web search tool)",
    "doc": "https://platform.claude.com/docs/en/agents-and-tools/tool-use/web-search-tool",
    "read": "2026-11-05",
    "price_source": "https://platform.claude.com/docs/en/agents-and-tools/tool-use/web-search-tool",
    "per_search": 0.01,
    "price_note": "$10 per 1,000 searches plus the tokens of the search results"
   },
   "gemini": {
    "how": "Gemini API generateContent with the google_search tool (Grounding with Google Search)",
    "doc": "https://ai.google.dev/gemini-api/docs/google-search",
    "read": "2026-11-05",
    "price_source": "https://ai.google.dev/gemini-api/docs/google-search",
    "per_search": 0.014,
    "price_note": "Gemini 3 and newer: $14 per 1,000 search queries the model runs, after 5,000 free a month (not deducted here)"
   },
   "xai": {
    "how": "xAI Responses API with the web_search tool",
    "doc": "https://docs.x.ai/developers/tools/web-search",
    "read": "2026-11-05",
    "price_source": "https://docs.x.ai/developers/pricing",
    "per_search": 0.005,
    "price_note": "$5 per 1,000 web_search calls plus tokens"
   },
   "openai-api": {
    "how": "Responses API with the {\"type\": \"web_search\"} tool (live web access, external_web_access left at its default, true)",
    "doc": "https://developers.openai.com/api/docs/guides/tools-web-search",
    "read": "2026-11-05",
    "price_source": "https://developers.openai.com/api/docs/pricing",
    "per_search": 0.01,
    "price_note": "$10 per 1,000 web search calls; search content tokens billed at the model's rates"
   },
   "claude-api": {
    "how": "Messages API with the {\"type\": \"web_search_20260318\", \"name\": \"web_search\"} tool, no max_uses; a pause_turn reply is continued",
    "doc": "https://platform.claude.com/docs/en/agents-and-tools/tool-use/web-search-tool",
    "read": "2026-11-05",
    "price_source": "https://platform.claude.com/docs/en/agents-and-tools/tool-use/web-search-tool",
    "per_search": 0.01,
    "price_note": "$10 per 1,000 searches plus the tokens of the search results"
   }
  },
  "web_not_run": "Kimi K3.1 Pro and GLM-5.3 run through OpenRouter, whose web search is OpenRouter's own plugin, not the makers' search; they run without web only."
 },
 "openai_web": {
  "guide": "https://developers.openai.com/api/docs/guides/decisions",
  "reference": "https://developers.openai.com/api/reference/resources/decisions/methods/create",
  "read": "2026-11-05",
  "parameters": [
   "model",
   "input",
   "questions",
   "prompt_cache_key",
   "prompt_cache_options",
   "safety_identifier"
  ]
 },
 "reported": {
  "perplexity": {
   "source": "https://www.perplexity.ai/hub/blog/web-grounding-for-the-decisions-api",
   "title": "Web grounding for the Decisions API",
   "published": "2026-11-04",
   "on_percent": 86,
   "off_percent": 51,
   "on_median_ms": 370,
   "on_p95_ms": 685
  },
  "parallel": {
   "source": "https://github.com/parallel-web/pws-decisions-benchmark",
   "evaluated": "2026-10-29",
   "rows": [
    {
     "name": "Parallel Decisions API (fast)",
     "public_percent": 78.33,
     "p50_s": 0.56,
     "p95_s": 0.71,
     "usd_per_1000": 1.0
    },
    {
     "name": "Perplexity Decisions (pplx-decider-v1.2, through OpenRouter)",
     "public_percent": 51.0
    },
    {
     "name": "OpenAI Decisions API (gpt-6-luna)",
     "public_percent": 42.0
    },
    {
     "name": "Baseten Mammoth (baseten-mammoth-27b-latest)",
     "public_percent": 37.0
    },
    {
     "name": "TypeSafe Jev 1.14 (jev-1.14.0)",
     "public_percent": 33.33
    }
   ],
   "random_percent": 44.67
  }
 },
 "failed_attempts": [
  {
   "setup": "gpt-6-astra",
   "attempt": "attempt-2",
   "run": 1,
   "rows": 100,
   "errors": 100,
   "correct": 0
  },
  {
   "setup": "gpt-6-astra",
   "attempt": "attempt-2",
   "run": 2,
   "rows": 100,
   "errors": 100,
   "correct": 0
  },
  {
   "setup": "gpt-6-astra",
   "attempt": "attempt-2",
   "run": 3,
   "rows": 100,
   "errors": 97,
   "correct": 3
  },
  {
   "setup": "gpt-6-astra",
   "attempt": "attempt-3",
   "run": 1,
   "rows": 100,
   "errors": 98,
   "correct": 1
  },
  {
   "setup": "gpt-6-astra",
   "attempt": "attempt-3",
   "run": 2,
   "rows": 100,
   "errors": 100,
   "correct": 0
  },
  {
   "setup": "gpt-6-astra",
   "attempt": "attempt-3",
   "run": 3,
   "rows": 5,
   "errors": 5,
   "correct": 0
  },
  {
   "setup": "gpt-6-astra",
   "attempt": "attempt-4",
   "run": 1,
   "rows": 100,
   "errors": 96,
   "correct": 2
  },
  {
   "setup": "gpt-6-astra",
   "attempt": "attempt-4",
   "run": 2,
   "rows": 32,
   "errors": 32,
   "correct": 0
  },
  {
   "setup": "gpt-6-astra-web",
   "attempt": "attempt-2",
   "run": 1,
   "rows": 100,
   "errors": 100,
   "correct": 0
  },
  {
   "setup": "gpt-6-astra-web",
   "attempt": "attempt-2",
   "run": 2,
   "rows": 100,
   "errors": 100,
   "correct": 0
  },
  {
   "setup": "gpt-6-astra-web",
   "attempt": "attempt-2",
   "run": 3,
   "rows": 100,
   "errors": 98,
   "correct": 2
  },
  {
   "setup": "gpt-6-astra-web",
   "attempt": "attempt-3",
   "run": 1,
   "rows": 100,
   "errors": 98,
   "correct": 2
  },
  {
   "setup": "gpt-6-astra-web",
   "attempt": "attempt-3",
   "run": 2,
   "rows": 100,
   "errors": 100,
   "correct": 0
  },
  {
   "setup": "gpt-6-astra-web",
   "attempt": "attempt-3",
   "run": 3,
   "rows": 10,
   "errors": 10,
   "correct": 0
  },
  {
   "setup": "gpt-6-astra-web",
   "attempt": "attempt-4",
   "run": 1,
   "rows": 100,
   "errors": 95,
   "correct": 5
  },
  {
   "setup": "gpt-6-astra-web",
   "attempt": "attempt-4",
   "run": 2,
   "rows": 47,
   "errors": 47,
   "correct": 0
  },
  {
   "setup": "gpt-6.1-sol-api",
   "attempt": "attempt-1",
   "run": 1,
   "rows": 100,
   "errors": 38,
   "correct": 46
  },
  {
   "setup": "gpt-6.1-sol-api",
   "attempt": "attempt-1",
   "run": 2,
   "rows": 100,
   "errors": 100,
   "correct": 0
  },
  {
   "setup": "gpt-6.1-sol-api",
   "attempt": "attempt-1",
   "run": 3,
   "rows": 100,
   "errors": 100,
   "correct": 0
  },
  {
   "setup": "gpt-6.1-sol-api-web",
   "attempt": "attempt-1",
   "run": 1,
   "rows": 100,
   "errors": 18,
   "correct": 82
  },
  {
   "setup": "gpt-6.1-sol-api-web",
   "attempt": "attempt-1",
   "run": 2,
   "rows": 100,
   "errors": 100,
   "correct": 0
  },
  {
   "setup": "gpt-6.1-sol-api-web",
   "attempt": "attempt-1",
   "run": 3,
   "rows": 100,
   "errors": 100,
   "correct": 0
  }
 ],
 "dont_know": {
  "option": "I don't know",
  "sentence": "If you do not know the answer, choose I don't know.",
  "provider": "perplexity",
  "model": "pplx-decider-v1.2",
  "dates": [
   "2026-11-05"
  ],
  "off": {
   "perplexity_mode": null,
   "runs": [
    {
     "started": "2026-11-05T17:52:08+00:00",
     "questions": 100,
     "right": 48,
     "wrong": 35,
     "dont_know": 17,
     "errors": 0,
     "accuracy": 0.48,
     "answered": 0.83,
     "wrong_of_answers": 0.42168674698795183,
     "non_hallucination": 0.3269230769230769
    },
    {
     "started": "2026-11-05T17:52:43+00:00",
     "questions": 100,
     "right": 48,
     "wrong": 36,
     "dont_know": 16,
     "errors": 0,
     "accuracy": 0.48,
     "answered": 0.84,
     "wrong_of_answers": 0.42857142857142855,
     "non_hallucination": 0.3076923076923077
    },
    {
     "started": "2026-11-05T17:53:28+00:00",
     "questions": 100,
     "right": 47,
     "wrong": 37,
     "dont_know": 16,
     "errors": 0,
     "accuracy": 0.47,
     "answered": 0.84,
     "wrong_of_answers": 0.44047619047619047,
     "non_hallucination": 0.3018867924528302
    }
   ],
   "right": {
    "mean": 47.666666666666664,
    "min": 47,
    "max": 48
   },
   "wrong": {
    "mean": 36,
    "min": 35,
    "max": 37
   },
   "dont_know": {
    "mean": 16.333333333333332,
    "min": 16,
    "max": 17
   },
   "errors": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "accuracy": {
    "mean": 0.4766666666666666,
    "min": 0.47,
    "max": 0.48
   },
   "answered": {
    "mean": 0.8366666666666667,
    "min": 0.83,
    "max": 0.84
   },
   "wrong_of_answers": {
    "mean": 0.4302447886785236,
    "min": 0.42168674698795183,
    "max": 0.44047619047619047
   },
   "non_hallucination": {
    "mean": 0.3121673923560716,
    "min": 0.3018867924528302,
    "max": 0.3269230769230769
   }
  },
  "on": {
   "perplexity_mode": "web_grounding",
   "runs": [
    {
     "started": "2026-11-05T17:54:08+00:00",
     "questions": 100,
     "right": 85,
     "wrong": 8,
     "dont_know": 7,
     "errors": 0,
     "accuracy": 0.85,
     "answered": 0.93,
     "wrong_of_answers": 0.08602150537634409,
     "non_hallucination": 0.4666666666666667
    },
    {
     "started": "2026-11-05T17:55:05+00:00",
     "questions": 100,
     "right": 84,
     "wrong": 9,
     "dont_know": 7,
     "errors": 0,
     "accuracy": 0.84,
     "answered": 0.93,
     "wrong_of_answers": 0.0967741935483871,
     "non_hallucination": 0.4375
    },
    {
     "started": "2026-11-05T17:55:59+00:00",
     "questions": 100,
     "right": 84,
     "wrong": 8,
     "dont_know": 8,
     "errors": 0,
     "accuracy": 0.84,
     "answered": 0.92,
     "wrong_of_answers": 0.08695652173913043,
     "non_hallucination": 0.5
    }
   ],
   "right": {
    "mean": 84.33333333333333,
    "min": 84,
    "max": 85
   },
   "wrong": {
    "mean": 8.333333333333334,
    "min": 8,
    "max": 9
   },
   "dont_know": {
    "mean": 7.333333333333333,
    "min": 7,
    "max": 8
   },
   "errors": {
    "mean": 0,
    "min": 0,
    "max": 0
   },
   "accuracy": {
    "mean": 0.8433333333333333,
    "min": 0.84,
    "max": 0.85
   },
   "answered": {
    "mean": 0.9266666666666667,
    "min": 0.92,
    "max": 0.93
   },
   "wrong_of_answers": {
    "mean": 0.08991740688795387,
    "min": 0.08602150537634409,
    "max": 0.0967741935483871
   },
   "non_hallucination": {
    "mean": 0.46805555555555556,
    "min": 0.4375,
    "max": 0.5
   }
  }
 }
}
