{
 "bench": "grading",
 "updated": "2026-11-16T19:44:24Z",
 "feed": {
  "file": "bench-scores.json",
  "sha256": "f2ebb5ed58d87c33daff7cf166553c5f6a70a4b176d6ba4802078b1d75ba82a5"
 },
 "cohort_pairs": 2069,
 "label_pairs": {
  "reference": 942,
  "other": 329,
  "none": 798
 },
 "false_alarm_pairs": 1740,
 "definitions": {
  "latency": "client-side wall time per request, request sent to answer parsed; compare only rows with the same requests_in_flight",
  "api_cost_per_1000_usd_list_price": "billed input tokens x the provider's list price (price_source, price_date), per 1,000 questions; for a chat model billed per token through OpenRouter, input and output (reasoning included) tokens at its list prices; not an invoice; null on own GPU and on a subscription route, and null with its rate, source and date where no public price page backs that exact model's rate (checked 13 Nov): unverified, not zero",
  "gpu_seconds_per_1000": "measured wall time of the whole run on one GPU (model load excluded) over all pairs it answered, per 1,000 questions: a rate, not a cohort share; own GPU only, not comparable with API dollars",
  "recall": "of all gold pairs with that label, the share labelled so; an invalid reply counts as missed",
  "wrong_answer_recall": "recall_other: texts stating a conflicting answer that were flagged 'other'",
  "false_alarm_rate": "gold 'reference' or 'none' pairs labelled 'other', over false_alarm_pairs; invalid replies are not counted as correct rejections, so 1 - false_alarm_rate is not an accuracy; read it beside invalid",
  "invalid": "replies without a usable label; for chat, no last matching ANSWER marker in the visible output under answer rule v2 (keyword in any letter case, option an uppercase A, B or C); counted wrong",
  "measurement_gap": "present only where a run was interrupted and resumed: latency and gpu_seconds_per_1000 are then null, with the reason; scores are unaffected",
  "probabilities": "this run's records hold per-option probabilities for every pair; says nothing about other runs of the model; no confidence is derived for chat models; calibration_error stays in bench-scores.json",
  "seed_groups": "a row with 'runs' covers three full runs with seeds 0, 1, 2 under its first run's id: recalls and false_alarm_rate are the mean of the three, latency is over all their requests, gpu_seconds_per_1000 the mean rate; 'invalid' is null and each seed's count is in invalid_per_seed"
 },
 "models": [
  {
   "run": "grading:d1-ultra-v1",
   "model_key": "liquid-d1-ultra",
   "latency_s_median": 0.485,
   "latency_s_p90": 0.903,
   "requests_in_flight": 8,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": 0.0351,
   "price_usd_per_million_input": 0.042,
   "price_source": "https://www.liquid.ai/blog/d1-ultra",
   "price_date": "2026-10-16",
   "gpu_seconds_per_1000": null,
   "recall_reference": 0.9565,
   "recall_other": 0.8085,
   "recall_none": 0.8659,
   "wrong_answer_recall": 0.8085,
   "false_alarm_rate": 0.0454,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:gemini-v1",
   "model_key": "gemini-decision-1",
   "latency_s_median": 0.203,
   "latency_s_p90": 0.291,
   "requests_in_flight": 8,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": 0.0935,
   "price_usd_per_million_input": 0.1,
   "price_source": "https://ai.google.dev/gemini-api/docs/pricing",
   "price_date": "2026-11-13",
   "gpu_seconds_per_1000": null,
   "recall_reference": 0.965,
   "recall_other": 0.5106,
   "recall_none": 0.8459,
   "wrong_answer_recall": 0.5106,
   "false_alarm_rate": 0.0178,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-liquid-d1-v1",
   "model_key": "liquid-d1",
   "latency_s_median": 0.265,
   "latency_s_p90": 0.601,
   "requests_in_flight": 8,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": null,
   "recall_reference": 0.9108,
   "recall_other": 0.6626,
   "recall_none": 0.8847,
   "wrong_answer_recall": 0.6626,
   "false_alarm_rate": 0.0529,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-liquid-d1.1-v1",
   "model_key": "liquid-d1.1",
   "latency_s_median": 0.25,
   "latency_s_p90": 0.453,
   "requests_in_flight": 8,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": 0.0167,
   "price_usd_per_million_input": 0.02,
   "price_source": "https://www.liquid.ai/pricing",
   "price_date": "2026-10-24",
   "gpu_seconds_per_1000": null,
   "recall_reference": 0.9246,
   "recall_other": 0.7052,
   "recall_none": 0.9085,
   "wrong_answer_recall": 0.7052,
   "false_alarm_rate": 0.0414,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-liquid-d1.2-ultra-v1",
   "model_key": "liquid-d1.2-ultra",
   "latency_s_median": 0.229,
   "latency_s_p90": 0.347,
   "requests_in_flight": 8,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": 0.0351,
   "price_usd_per_million_input": 0.042,
   "price_source": "https://www.liquid.ai/blog/d1-2",
   "price_date": "2026-11-16",
   "gpu_seconds_per_1000": null,
   "recall_reference": 0.9193,
   "recall_other": 0.8541,
   "recall_none": 0.9211,
   "wrong_answer_recall": 0.8541,
   "false_alarm_rate": 0.0443,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-fastino-glide-v1",
   "model_key": "fastino-glide",
   "latency_s_median": 0.299,
   "latency_s_p90": 0.558,
   "requests_in_flight": 8,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": null,
   "recall_reference": 0.9639,
   "recall_other": 0.6869,
   "recall_none": 0.9135,
   "wrong_answer_recall": 0.6869,
   "false_alarm_rate": 0.0172,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-subq-snap-1.1-v1",
   "model_key": "subq-snap-1.1",
   "latency_s_median": 0.394,
   "latency_s_p90": 0.412,
   "requests_in_flight": 8,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": null,
   "recall_reference": 0.9193,
   "recall_other": 0.7812,
   "recall_none": 0.9248,
   "wrong_answer_recall": 0.7812,
   "false_alarm_rate": 0.0414,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-jev-1.13.0-v1",
   "model_key": "jev-1.13.0",
   "latency_s_median": 0.169,
   "latency_s_p90": 0.19,
   "requests_in_flight": 8,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": 0.0487,
   "price_usd_per_million_input": 0.042,
   "price_source": "https://typesafe.ai/blog/introducing-system-one-models-and-jev",
   "price_date": "2026-10-08",
   "gpu_seconds_per_1000": null,
   "recall_reference": 0.9427,
   "recall_other": 0.7994,
   "recall_none": 0.8559,
   "wrong_answer_recall": 0.7994,
   "false_alarm_rate": 0.0598,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-jev-1.14.0-v1",
   "model_key": "jev-1.14.0",
   "latency_s_median": 0.176,
   "latency_s_p90": 0.203,
   "requests_in_flight": 8,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": 0.0487,
   "price_usd_per_million_input": 0.042,
   "price_source": "https://docs.typesafe.ai/models",
   "price_date": "2026-10-16",
   "gpu_seconds_per_1000": null,
   "recall_reference": 0.9406,
   "recall_other": 0.7204,
   "recall_none": 0.9023,
   "wrong_answer_recall": 0.7204,
   "false_alarm_rate": 0.0351,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-microsoft-decision-1-v1",
   "model_key": "microsoft-decision-1",
   "latency_s_median": 0.324,
   "latency_s_p90": 0.377,
   "requests_in_flight": 8,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": 0.0363,
   "price_usd_per_million_input": 0.042,
   "price_source": "https://openrouter.ai/microsoft/microsoft-decision-1",
   "price_date": "2026-10-17",
   "gpu_seconds_per_1000": null,
   "recall_reference": 0.8949,
   "recall_other": 0.8237,
   "recall_none": 0.9185,
   "wrong_answer_recall": 0.8237,
   "false_alarm_rate": 0.0649,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-meragpt-decider-1-v1",
   "model_key": "meragpt-decider-1",
   "latency_s_median": 0.652,
   "latency_s_p90": 0.754,
   "requests_in_flight": 8,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": 0.0353,
   "price_usd_per_million_input": 0.04,
   "price_source": "https://meragpt.com/docs/models",
   "price_date": "2026-10-19",
   "gpu_seconds_per_1000": null,
   "recall_reference": 0.7017,
   "recall_other": 0.3191,
   "recall_none": 0.6529,
   "wrong_answer_recall": 0.3191,
   "false_alarm_rate": 0.1759,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-liquid-d1-2b-v1",
   "model_key": "liquid-d1-2b",
   "latency_s_median": 0.019,
   "latency_s_p90": 0.025,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 25.4,
   "recall_reference": 0.8429,
   "recall_other": 0.8571,
   "recall_none": 0.3734,
   "wrong_answer_recall": 0.8571,
   "false_alarm_rate": 0.3218,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-glide-2b-v1",
   "model_key": "glide-2b",
   "latency_s_median": 0.035,
   "latency_s_p90": 0.041,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 43.5,
   "recall_reference": 0.897,
   "recall_other": 0.6079,
   "recall_none": 0.6892,
   "wrong_answer_recall": 0.6079,
   "false_alarm_rate": 0.108,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-knowline-31b-gen1.5-v1",
   "model_key": "knowline-31b-gen1.5",
   "latency_s_median": 2.545,
   "latency_s_p90": 3.881,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 2613.4,
   "recall_reference": 0.9724,
   "recall_other": 0.8359,
   "recall_none": 0.9073,
   "wrong_answer_recall": 0.8359,
   "false_alarm_rate": 0.0305,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-glide-27b-v1",
   "model_key": "glide-27b",
   "latency_s_median": 1.035,
   "latency_s_p90": 1.951,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 1204.8,
   "recall_reference": 0.9713,
   "recall_other": 0.7477,
   "recall_none": 0.8972,
   "wrong_answer_recall": 0.7477,
   "false_alarm_rate": 0.0149,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-sieve-31b-v3-v1",
   "model_key": "sieve-31b-v3",
   "latency_s_median": 2.673,
   "latency_s_p90": 3.931,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 2823.7,
   "recall_reference": 0.9862,
   "recall_other": 0.7933,
   "recall_none": 0.896,
   "wrong_answer_recall": 0.7933,
   "false_alarm_rate": 0.0213,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-fastino-glide-v1.2-v1",
   "model_key": "fastino-glide-v1.2",
   "latency_s_median": 0.234,
   "latency_s_p90": 0.357,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 253.6,
   "recall_reference": 0.966,
   "recall_other": 0.693,
   "recall_none": 0.906,
   "wrong_answer_recall": 0.693,
   "false_alarm_rate": 0.0184,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-reax-1.3-v1",
   "model_key": "reax-1.3",
   "latency_s_median": 0.677,
   "latency_s_p90": 1.047,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 745.3,
   "recall_reference": 0.9618,
   "recall_other": 0.6748,
   "recall_none": 0.9536,
   "wrong_answer_recall": 0.6748,
   "false_alarm_rate": 0.0121,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-turion-jev-31b-v1",
   "model_key": "turion-jev-31b",
   "latency_s_median": 1.123,
   "latency_s_p90": 1.956,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 1328.9,
   "recall_reference": 0.9682,
   "recall_other": 0.6626,
   "recall_none": 0.9424,
   "wrong_answer_recall": 0.6626,
   "false_alarm_rate": 0.0167,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-perplexity-decider-27b-v1.3-v1",
   "model_key": "perplexity-decider-27b-v1.3",
   "latency_s_median": 0.223,
   "latency_s_p90": 0.34,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 245.0,
   "recall_reference": 0.9512,
   "recall_other": 0.8267,
   "recall_none": 0.891,
   "wrong_answer_recall": 0.8267,
   "false_alarm_rate": 0.0351,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-perplexity-decider-flash-v1.3-v1",
   "model_key": "perplexity-decider-flash-v1.3",
   "latency_s_median": 0.055,
   "latency_s_p90": 0.081,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 61.2,
   "recall_reference": 0.9374,
   "recall_other": 0.6748,
   "recall_none": 0.8421,
   "wrong_answer_recall": 0.6748,
   "false_alarm_rate": 0.0557,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-perplexity-decider-flash-v1.2-v1",
   "model_key": "perplexity-decider-flash-v1.2",
   "latency_s_median": 0.058,
   "latency_s_p90": 0.085,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 64.6,
   "recall_reference": 0.9406,
   "recall_other": 0.4772,
   "recall_none": 0.891,
   "wrong_answer_recall": 0.4772,
   "false_alarm_rate": 0.0397,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-glide-9b-v1",
   "model_key": "glide-9b",
   "latency_s_median": 0.08,
   "latency_s_p90": 0.113,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 90.4,
   "recall_reference": 0.9459,
   "recall_other": 0.6991,
   "recall_none": 0.881,
   "wrong_answer_recall": 0.6991,
   "false_alarm_rate": 0.0379,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-elephant-1-v1",
   "model_key": "elephant-1",
   "latency_s_median": 0.09,
   "latency_s_p90": 0.111,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 98.0,
   "recall_reference": 0.9374,
   "recall_other": 0.4103,
   "recall_none": 0.9173,
   "wrong_answer_recall": 0.4103,
   "false_alarm_rate": 0.019,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-karpo-decision-31b-v1",
   "model_key": "karpo-decision-31b",
   "latency_s_median": 0.216,
   "latency_s_p90": 0.311,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 230.5,
   "recall_reference": 0.9798,
   "recall_other": 0.8085,
   "recall_none": 0.9298,
   "wrong_answer_recall": 0.8085,
   "false_alarm_rate": 0.0167,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-copernicus-31b-v1",
   "model_key": "copernicus-31b",
   "latency_s_median": 0.516,
   "latency_s_p90": 0.829,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 577.0,
   "recall_reference": 0.9883,
   "recall_other": 0.8146,
   "recall_none": 0.8997,
   "wrong_answer_recall": 0.8146,
   "false_alarm_rate": 0.0253,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-knowline-9b-gen1-v1",
   "model_key": "knowline-9b-gen1",
   "latency_s_median": 0.097,
   "latency_s_p90": 0.143,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 104.7,
   "recall_reference": 0.8439,
   "recall_other": 0.5076,
   "recall_none": 0.9749,
   "wrong_answer_recall": 0.5076,
   "false_alarm_rate": 0.0379,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-adjudex-9b-v5-v1",
   "model_key": "adjudex-9b-v5",
   "latency_s_median": 0.065,
   "latency_s_p90": 0.092,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 71.4,
   "recall_reference": 0.9225,
   "recall_other": 0.6505,
   "recall_none": 0.8972,
   "wrong_answer_recall": 0.6505,
   "false_alarm_rate": 0.0489,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-knowline-4b-gen5.5-v1",
   "model_key": "knowline-4b-gen5.5",
   "latency_s_median": 0.045,
   "latency_s_p90": 0.064,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 49.0,
   "recall_reference": 0.9055,
   "recall_other": 0.6201,
   "recall_none": 0.9336,
   "wrong_answer_recall": 0.6201,
   "false_alarm_rate": 0.0339,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-knowline-9b-gen2-v1",
   "model_key": "knowline-9b-gen2",
   "latency_s_median": 0.101,
   "latency_s_p90": 0.158,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 128.5,
   "recall_reference": 0.8928,
   "recall_other": 0.3587,
   "recall_none": 0.9637,
   "wrong_answer_recall": 0.3587,
   "false_alarm_rate": 0.0144,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-knowline-12b-gen1-v1",
   "model_key": "knowline-12b-gen1",
   "latency_s_median": 0.163,
   "latency_s_p90": 0.247,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 194.6,
   "recall_reference": 0.9331,
   "recall_other": 0.6109,
   "recall_none": 0.9085,
   "wrong_answer_recall": 0.6109,
   "false_alarm_rate": 0.0356,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-knowline-31b-gen2-v1",
   "model_key": "knowline-31b-gen2",
   "latency_s_median": 0.389,
   "latency_s_p90": 0.612,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 449.4,
   "recall_reference": 0.9682,
   "recall_other": 0.8632,
   "recall_none": 0.881,
   "wrong_answer_recall": 0.8632,
   "false_alarm_rate": 0.0362,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-jeffan-9b-1211-v1",
   "model_key": "jeffan-9b-1211",
   "latency_s_median": 0.093,
   "latency_s_p90": 0.138,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 99.6,
   "recall_reference": 0.9172,
   "recall_other": 0.5988,
   "recall_none": 0.9687,
   "wrong_answer_recall": 0.5988,
   "false_alarm_rate": 0.0247,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-cerno-9b-v1.2-v1",
   "model_key": "cerno-9b-v1.2",
   "latency_s_median": 0.114,
   "latency_s_p90": 0.177,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 126.3,
   "recall_reference": 0.9119,
   "recall_other": 0.6535,
   "recall_none": 0.9336,
   "wrong_answer_recall": 0.6535,
   "false_alarm_rate": 0.0305,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-chinese-decision-pointer-4b-v1",
   "model_key": "chinese-decision-pointer-4b",
   "latency_s_median": 0.069,
   "latency_s_p90": 0.107,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 76.8,
   "recall_reference": 0.9066,
   "recall_other": 0.69,
   "recall_none": 0.8596,
   "wrong_answer_recall": 0.69,
   "false_alarm_rate": 0.0603,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-logits-sigma-1-duo-v1",
   "model_key": "logits-sigma-1-duo",
   "latency_s_median": 0.641,
   "latency_s_p90": 1.014,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 723.3,
   "recall_reference": 0.9437,
   "recall_other": 0.6383,
   "recall_none": 0.9586,
   "wrong_answer_recall": 0.6383,
   "false_alarm_rate": 0.0086,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-startlux-decision-v1.1-4b-v1",
   "model_key": "startlux-decision-v1.1-4b",
   "latency_s_median": 0.121,
   "latency_s_p90": 0.186,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 140.3,
   "recall_reference": 0.9023,
   "recall_other": 0.8936,
   "recall_none": 0.8421,
   "wrong_answer_recall": 0.8936,
   "false_alarm_rate": 0.0828,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-startlux-decision-v1.1-9b-v1",
   "model_key": "startlux-decision-v1.1-9b",
   "latency_s_median": 0.229,
   "latency_s_p90": 0.357,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 267.5,
   "recall_reference": 0.9034,
   "recall_other": 0.8875,
   "recall_none": 0.7945,
   "wrong_answer_recall": 0.8875,
   "false_alarm_rate": 0.0954,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-verdin-4b-v1.1-v1",
   "model_key": "verdin-4b-v1.1",
   "latency_s_median": 0.049,
   "latency_s_p90": 0.065,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 54.1,
   "recall_reference": 0.9501,
   "recall_other": 0.4681,
   "recall_none": 0.8521,
   "wrong_answer_recall": 0.4681,
   "false_alarm_rate": 0.0374,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-valen-decision-1.5-9b-v1",
   "model_key": "valen-decision-1.5-9b",
   "latency_s_median": 0.071,
   "latency_s_p90": 0.095,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 78.4,
   "recall_reference": 0.9342,
   "recall_other": 0.3982,
   "recall_none": 0.881,
   "wrong_answer_recall": 0.3982,
   "false_alarm_rate": 0.0276,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-vera-medium-omni-v1",
   "model_key": "vera-medium-omni",
   "latency_s_median": 0.057,
   "latency_s_p90": 0.063,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 59.6,
   "recall_reference": 0.9278,
   "recall_other": 0.7599,
   "recall_none": 0.9023,
   "wrong_answer_recall": 0.7599,
   "false_alarm_rate": 0.0569,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-dexy-decision-9b-v2-v1",
   "model_key": "dexy-decision-9b-v2",
   "latency_s_median": 0.063,
   "latency_s_p90": 0.088,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 71.5,
   "recall_reference": 0.9268,
   "recall_other": 0.6474,
   "recall_none": 0.9048,
   "wrong_answer_recall": 0.6474,
   "false_alarm_rate": 0.0391,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-clef-flash-v1",
   "model_key": "clef-flash",
   "latency_s_median": 0.152,
   "latency_s_p90": 0.18,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 155.7,
   "recall_reference": 0.9257,
   "recall_other": 0.3769,
   "recall_none": 0.8221,
   "wrong_answer_recall": 0.3769,
   "false_alarm_rate": 0.0391,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-cygnet-v1",
   "model_key": "cygnet",
   "latency_s_median": 0.087,
   "latency_s_p90": 0.128,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 92.9,
   "recall_reference": 0.9437,
   "recall_other": 0.2918,
   "recall_none": 0.8897,
   "wrong_answer_recall": 0.2918,
   "false_alarm_rate": 0.0207,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-winnow-12b-v1",
   "model_key": "winnow-12b",
   "latency_s_median": 0.122,
   "latency_s_p90": 0.178,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 132.1,
   "recall_reference": 0.9427,
   "recall_other": 0.5167,
   "recall_none": 0.8622,
   "wrong_answer_recall": 0.5167,
   "false_alarm_rate": 0.0379,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-metask-jev-4b-v1",
   "model_key": "metask-jev-4b",
   "latency_s_median": 0.116,
   "latency_s_p90": 0.186,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 133.9,
   "recall_reference": 0.9618,
   "recall_other": 0.231,
   "recall_none": 0.7005,
   "wrong_answer_recall": 0.231,
   "false_alarm_rate": 0.0339,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-strands-decider-2b-v1",
   "model_key": "strands-decider-2b",
   "latency_s_median": 0.038,
   "latency_s_p90": 0.054,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 43.1,
   "recall_reference": 0.9639,
   "recall_other": 0.0456,
   "recall_none": 0.7506,
   "wrong_answer_recall": 0.0456,
   "false_alarm_rate": 0.008,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-glide-0.8b-v1",
   "model_key": "glide-0.8b",
   "latency_s_median": 0.048,
   "latency_s_p90": 0.075,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 63.3,
   "recall_reference": 0.8992,
   "recall_other": 0.3131,
   "recall_none": 0.7243,
   "wrong_answer_recall": 0.3131,
   "false_alarm_rate": 0.0655,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-glide-0.4b-v1",
   "model_key": "glide-0.4b",
   "latency_s_median": 0.028,
   "latency_s_p90": 0.044,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 40.7,
   "recall_reference": 0.8397,
   "recall_other": 0.31,
   "recall_none": 0.5125,
   "wrong_answer_recall": 0.31,
   "false_alarm_rate": 0.1684,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-startlux-decision-v1.1-2b-v1",
   "model_key": "startlux-decision-v1.1-2b",
   "latency_s_median": 0.063,
   "latency_s_p90": 0.087,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 78.2,
   "recall_reference": 0.8875,
   "recall_other": 0.6748,
   "recall_none": 0.6792,
   "wrong_answer_recall": 0.6748,
   "false_alarm_rate": 0.1397,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-startlux-decision-v1.1-0.8b-instant-v1",
   "model_key": "startlux-decision-v1.1-0.8b-instant",
   "latency_s_median": 0.042,
   "latency_s_p90": 0.059,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 53.3,
   "recall_reference": 0.8386,
   "recall_other": 0.7234,
   "recall_none": 0.4211,
   "wrong_answer_recall": 0.7234,
   "false_alarm_rate": 0.2649,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-onejev1.6-0.8b-v1",
   "model_key": "onejev1.6-0.8b",
   "latency_s_median": 0.031,
   "latency_s_p90": 0.043,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 42.9,
   "recall_reference": 0.8227,
   "recall_other": 0.3343,
   "recall_none": 0.6203,
   "wrong_answer_recall": 0.3343,
   "false_alarm_rate": 0.1437,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-knowline-2b-gen1-v1",
   "model_key": "knowline-2b-gen1",
   "latency_s_median": 0.109,
   "latency_s_p90": 0.17,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 129.8,
   "recall_reference": 0.9257,
   "recall_other": 0.1641,
   "recall_none": 0.7594,
   "wrong_answer_recall": 0.1641,
   "false_alarm_rate": 0.027,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-knowline-0.8b-gen1-v1",
   "model_key": "knowline-0.8b-gen1",
   "latency_s_median": 0.068,
   "latency_s_p90": 0.103,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 81.2,
   "recall_reference": 0.9798,
   "recall_other": 0.1003,
   "recall_none": 0.3321,
   "wrong_answer_recall": 0.1003,
   "false_alarm_rate": 0.0356,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:site-gliclass-v4-edge-v1",
   "model_key": "gliclass-v4-edge",
   "latency_s_median": 0.075,
   "latency_s_p90": 0.18,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 128.5,
   "recall_reference": 0.9246,
   "recall_other": 0.3495,
   "recall_none": 0.3947,
   "wrong_answer_recall": 0.3495,
   "false_alarm_rate": 0.1057,
   "invalid": 0,
   "probabilities": true
  },
  {
   "run": "grading:chat-qwen3.5-4b-nothink-v1",
   "model_key": "qwen3.5-4b-nothink",
   "runs": [
    "grading:chat-qwen3.5-4b-nothink-v1",
    "grading:chat-qwen3.5-4b-nothink-seed1-v1",
    "grading:chat-qwen3.5-4b-nothink-seed2-v1"
   ],
   "aggregation": "mean of 3 seeds; latency over all requests of the three runs",
   "latency_s_median": 3.487,
   "latency_s_p90": 20.434,
   "requests_in_flight": 64,
   "requests_measured": 6207,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 148.4,
   "recall_reference": 0.9441,
   "recall_other": 0.612,
   "recall_none": 0.6145,
   "wrong_answer_recall": 0.612,
   "false_alarm_rate": 0.0764,
   "invalid": null,
   "invalid_per_seed": [
    20,
    25,
    15
   ],
   "probabilities": false
  },
  {
   "run": "grading:chat-qwen3.5-2b-nothink-v1",
   "model_key": "qwen3.5-2b-nothink",
   "runs": [
    "grading:chat-qwen3.5-2b-nothink-v1",
    "grading:chat-qwen3.5-2b-nothink-seed1-v1",
    "grading:chat-qwen3.5-2b-nothink-seed2-v1"
   ],
   "aggregation": "mean of 3 seeds; latency over all requests of the three runs",
   "latency_s_median": 1.53,
   "latency_s_p90": 13.725,
   "requests_in_flight": 64,
   "requests_measured": 6207,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 91.6,
   "recall_reference": 0.804,
   "recall_other": 0.3445,
   "recall_none": 0.6416,
   "wrong_answer_recall": 0.3445,
   "false_alarm_rate": 0.0776,
   "invalid": null,
   "invalid_per_seed": [
    37,
    45,
    43
   ],
   "probabilities": false
  },
  {
   "run": "grading:chat-gemma-4-e4b-think-v1",
   "model_key": "gemma-4-e4b-think",
   "latency_s_median": 9.38,
   "latency_s_p90": 11.907,
   "requests_in_flight": 30,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 325.5,
   "recall_reference": 0.9119,
   "recall_other": 0.6748,
   "recall_none": 0.8434,
   "wrong_answer_recall": 0.6748,
   "false_alarm_rate": 0.0448,
   "invalid": 2,
   "probabilities": false
  },
  {
   "run": "grading:chat-raon-1-nano-nothink-v1",
   "model_key": "raon-1-nano-nothink",
   "runs": [
    "grading:chat-raon-1-nano-nothink-v1",
    "grading:chat-raon-1-nano-nothink-seed1-v1",
    "grading:chat-raon-1-nano-nothink-seed2-v1"
   ],
   "aggregation": "mean of 3 seeds; latency over all requests of the three runs",
   "latency_s_median": 3.029,
   "latency_s_p90": 7.765,
   "requests_in_flight": 30,
   "requests_measured": 6207,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 208.6,
   "recall_reference": 0.649,
   "recall_other": 0.1986,
   "recall_none": 0.7189,
   "wrong_answer_recall": 0.1986,
   "false_alarm_rate": 0.0203,
   "invalid": null,
   "invalid_per_seed": [
    0,
    1,
    1
   ],
   "probabilities": false
  },
  {
   "run": "grading:chat-granite-4.2-3b-think-v1",
   "model_key": "granite-4.2-3b-think",
   "latency_s_median": 19.592,
   "latency_s_p90": 64.465,
   "requests_in_flight": 30,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 1050.3,
   "recall_reference": 0.8832,
   "recall_other": 0.7903,
   "recall_none": 0.807,
   "wrong_answer_recall": 0.7903,
   "false_alarm_rate": 0.0902,
   "invalid": 0,
   "probabilities": false
  },
  {
   "run": "grading:chat-lfm2.5-8b-a1b-think-v1",
   "model_key": "lfm2.5-8b-a1b-think",
   "latency_s_median": 9.579,
   "latency_s_p90": 15.194,
   "requests_in_flight": 30,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 351.9,
   "recall_reference": 0.7951,
   "recall_other": 0.0881,
   "recall_none": 0.8872,
   "wrong_answer_recall": 0.0881,
   "false_alarm_rate": 0.008,
   "invalid": 0,
   "probabilities": false
  },
  {
   "run": "grading:chat-gemini-4-hydrogen-v1",
   "model_key": "gemini-4-hydrogen",
   "latency_s_median": 6.372,
   "latency_s_p90": 19.169,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": null,
   "recall_reference": 0.8843,
   "recall_other": 0.6869,
   "recall_none": 0.9837,
   "wrong_answer_recall": 0.6869,
   "false_alarm_rate": 0.0218,
   "invalid": 0,
   "probabilities": false
  },
  {
   "run": "grading:chat-inkling-2-small-v1",
   "model_key": "inkling-2-small",
   "latency_s_median": 2.702,
   "latency_s_p90": 16.351,
   "requests_in_flight": 1,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": 1.4933,
   "price_usd_per_million_input": 0.3,
   "price_source": "https://openrouter.ai/api/v1/models",
   "price_date": "2026-11-16",
   "gpu_seconds_per_1000": null,
   "recall_reference": 0.9724,
   "recall_other": 0.7356,
   "recall_none": 0.9035,
   "wrong_answer_recall": 0.7356,
   "false_alarm_rate": 0.0241,
   "invalid": 0,
   "probabilities": false
  },
  {
   "run": "grading:chat-lfm3-1.7b-seed0-replay-v1",
   "model_key": "lfm3-1.7b",
   "runs": [
    "grading:chat-lfm3-1.7b-seed0-replay-v1",
    "grading:chat-lfm3-1.7b-seed1-r2-v1",
    "grading:chat-lfm3-1.7b-seed2-r2-v1"
   ],
   "aggregation": "mean of 3 seeds; latency over all requests of the three runs",
   "latency_s_median": 5.568,
   "latency_s_p90": 16.417,
   "requests_in_flight": 30,
   "requests_measured": 6207,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 268.3,
   "recall_reference": 0.9176,
   "recall_other": 0.6332,
   "recall_none": 0.83,
   "wrong_answer_recall": 0.6332,
   "false_alarm_rate": 0.041,
   "invalid": null,
   "invalid_per_seed": [
    2,
    1,
    1
   ],
   "probabilities": false
  },
  {
   "run": "grading:chat-minicpm5-2b-seed0-r2-v1",
   "model_key": "minicpm5-2b",
   "runs": [
    "grading:chat-minicpm5-2b-seed0-r2-v1",
    "grading:chat-minicpm5-2b-seed1-r2-v1",
    "grading:chat-minicpm5-2b-seed2-r2-v1"
   ],
   "aggregation": "mean of 3 seeds; latency over all requests of the three runs",
   "latency_s_median": 4.997,
   "latency_s_p90": 8.723,
   "requests_in_flight": 30,
   "requests_measured": 6207,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 196.7,
   "recall_reference": 0.8984,
   "recall_other": 0.7741,
   "recall_none": 0.8542,
   "wrong_answer_recall": 0.7741,
   "false_alarm_rate": 0.0646,
   "invalid": null,
   "invalid_per_seed": [
    3,
    0,
    1
   ],
   "probabilities": false
  },
  {
   "run": "grading:chat-gemma-4-12b-r2-v1",
   "model_key": "gemma-4-12b",
   "latency_s_median": 5.121,
   "latency_s_p90": 15.745,
   "requests_in_flight": 64,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 120.9,
   "recall_reference": 0.9735,
   "recall_other": 0.5198,
   "recall_none": 0.8321,
   "wrong_answer_recall": 0.5198,
   "false_alarm_rate": 0.0264,
   "invalid": 0,
   "probabilities": false
  },
  {
   "run": "grading:chat-lfm3-1.7b-nothink-r2-v1",
   "model_key": "lfm3-1.7b-nothink",
   "runs": [
    "grading:chat-lfm3-1.7b-nothink-r2-v1",
    "grading:chat-lfm3-1.7b-nothink-seed1-r2-v1",
    "grading:chat-lfm3-1.7b-nothink-seed2-r2-v1"
   ],
   "aggregation": "mean of 3 seeds; latency over all requests of the three runs",
   "latency_s_median": 7.618,
   "latency_s_p90": 22.257,
   "requests_in_flight": 64,
   "requests_measured": 6207,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 179.9,
   "recall_reference": 0.9211,
   "recall_other": 0.6261,
   "recall_none": 0.8133,
   "wrong_answer_recall": 0.6261,
   "false_alarm_rate": 0.0387,
   "invalid": null,
   "invalid_per_seed": [
    2,
    2,
    1
   ],
   "probabilities": false
  },
  {
   "run": "grading:chat-gemma-4-e2b-think-r2-v1",
   "model_key": "gemma-4-e2b-think",
   "latency_s_median": 4.022,
   "latency_s_p90": 5.851,
   "requests_in_flight": 64,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 69.4,
   "recall_reference": 0.896,
   "recall_other": 0.4954,
   "recall_none": 0.8321,
   "wrong_answer_recall": 0.4954,
   "false_alarm_rate": 0.0511,
   "invalid": 0,
   "probabilities": false
  },
  {
   "run": "grading:chat-qwen3.5-2b-r2-v1",
   "model_key": "qwen3.5-2b",
   "latency_s_median": 36.808,
   "latency_s_p90": 66.933,
   "requests_in_flight": 64,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 635.8,
   "recall_reference": 0.8652,
   "recall_other": 0.5532,
   "recall_none": 0.8308,
   "wrong_answer_recall": 0.5532,
   "false_alarm_rate": 0.0431,
   "invalid": 78,
   "probabilities": false
  },
  {
   "run": "grading:chat-minicpm5-2b-nothink-r2-v1",
   "model_key": "minicpm5-2b-nothink",
   "runs": [
    "grading:chat-minicpm5-2b-nothink-r2-v1",
    "grading:chat-minicpm5-2b-nothink-seed1-r2-v1",
    "grading:chat-minicpm5-2b-nothink-seed2-r2-v1"
   ],
   "aggregation": "mean of 3 seeds; latency over all requests of the three runs",
   "latency_s_median": 0.858,
   "latency_s_p90": 3.485,
   "requests_in_flight": 64,
   "requests_measured": 6207,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 51.3,
   "recall_reference": 0.8793,
   "recall_other": 0.2877,
   "recall_none": 0.8655,
   "wrong_answer_recall": 0.2877,
   "false_alarm_rate": 0.0272,
   "invalid": null,
   "invalid_per_seed": [
    6,
    5,
    4
   ],
   "probabilities": false
  },
  {
   "run": "grading:chat-gemma-4-12b-think-r2-v1",
   "model_key": "gemma-4-12b-think",
   "latency_s_median": 32.568,
   "latency_s_p90": 1640.171,
   "requests_in_flight": 64,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 4327.1,
   "recall_reference": 0.9342,
   "recall_other": 0.3799,
   "recall_none": 0.7744,
   "wrong_answer_recall": 0.3799,
   "false_alarm_rate": 0.0069,
   "invalid": 296,
   "probabilities": false
  },
  {
   "run": "grading:chat-qwen3.5-4b-r2-v1",
   "model_key": "qwen3.5-4b",
   "latency_s_median": 37.828,
   "latency_s_p90": 96.135,
   "requests_in_flight": 64,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 740.9,
   "recall_reference": 0.8769,
   "recall_other": 0.7325,
   "recall_none": 0.6717,
   "wrong_answer_recall": 0.7325,
   "false_alarm_rate": 0.0615,
   "invalid": 179,
   "probabilities": false
  },
  {
   "run": "grading:chat-gemma-4-e2b-r2-v1",
   "model_key": "gemma-4-e2b",
   "latency_s_median": 0.194,
   "latency_s_p90": 3.314,
   "requests_in_flight": 64,
   "requests_measured": 2069,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 18.3,
   "recall_reference": 0.9469,
   "recall_other": 0.0608,
   "recall_none": 0.6679,
   "wrong_answer_recall": 0.0608,
   "false_alarm_rate": 0.0132,
   "invalid": 0,
   "probabilities": false
  },
  {
   "run": "grading:chat-gemma-4-e2b-sampled-seed0-v1",
   "model_key": "gemma-4-e2b-sampled",
   "runs": [
    "grading:chat-gemma-4-e2b-sampled-seed0-v1",
    "grading:chat-gemma-4-e2b-sampled-seed1-v1",
    "grading:chat-gemma-4-e2b-sampled-seed2-v1"
   ],
   "aggregation": "mean of 3 seeds; latency over all requests of the three runs",
   "latency_s_median": 0.176,
   "latency_s_p90": 5.421,
   "requests_in_flight": 64,
   "requests_measured": 6207,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 33.0,
   "recall_reference": 0.9427,
   "recall_other": 0.077,
   "recall_none": 0.6734,
   "wrong_answer_recall": 0.077,
   "false_alarm_rate": 0.0155,
   "invalid": null,
   "invalid_per_seed": [
    1,
    0,
    1
   ],
   "probabilities": false
  },
  {
   "run": "grading:chat-gemma-4-e4b-think-sampled-seed0-v1",
   "model_key": "gemma-4-e4b-think-sampled",
   "runs": [
    "grading:chat-gemma-4-e4b-think-sampled-seed0-v1",
    "grading:chat-gemma-4-e4b-think-sampled-seed1-v1",
    "grading:chat-gemma-4-e4b-think-sampled-seed2-v1"
   ],
   "aggregation": "mean of 3 seeds; latency over all requests of the three runs",
   "latency_s_median": 6.135,
   "latency_s_p90": 9.7,
   "requests_in_flight": 30,
   "requests_measured": 6207,
   "api_cost_per_1000_usd_list_price": null,
   "price_usd_per_million_input": null,
   "price_source": null,
   "price_date": null,
   "gpu_seconds_per_1000": 233.6,
   "recall_reference": 0.9105,
   "recall_other": 0.6434,
   "recall_none": 0.8225,
   "wrong_answer_recall": 0.6434,
   "false_alarm_rate": 0.0427,
   "invalid": null,
   "invalid_per_seed": [
    34,
    31,
    30
   ],
   "probabilities": false
  }
 ]
}
