{
 "dataset": "league-2",
 "decisions": 39538,
 "finished": "2026-10-26T15:46:30.345Z",
 "games": 954,
 "id": 2,
 "matches": 171,
 "matches_done": 171,
 "matches_list": [
  {
   "games": 6,
   "match_id": 118,
   "matchday": 1,
   "model1": "Gemini 3.8 Flash (high)",
   "model2": "GLM-5.3 (low)",
   "score1": 5,
   "score2": 3,
   "status": "finished"
  },
  {
   "games": 8,
   "match_id": 124,
   "matchday": 2,
   "model1": "Gemini 3.8 Flash (high)",
   "model2": "Claude Sonnet 5.5 (low)",
   "score1": 5,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 121,
   "matchday": 3,
   "model1": "GNUbg weak",
   "model2": "Gemini 3.8 Flash (high)",
   "score1": 5,
   "score2": 0,
   "status": "finished"
  },
  {
   "games": 8,
   "match_id": 112,
   "matchday": 3,
   "model1": "Claude Sonnet 5.5 (high)",
   "model2": "GLM-5.3 (low)",
   "score1": 5,
   "score2": 3,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 122,
   "matchday": 4,
   "model1": "Gemini 3.8 Flash (high)",
   "model2": "DeepSeek-V4.1-Flash (low)",
   "score1": 3,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 111,
   "matchday": 4,
   "model1": "Claude Sonnet 5.5 (high)",
   "model2": "Claude Sonnet 5.5 (low)",
   "score1": 4,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 104,
   "matchday": 5,
   "model1": "GLM-5.3 (low)",
   "model2": "Claude Sonnet 5.5 (low)",
   "score1": 0,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 102,
   "matchday": 5,
   "model1": "GNUbg weak",
   "model2": "Claude Sonnet 5.5 (high)",
   "score1": 0,
   "score2": 8,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 138,
   "matchday": 6,
   "model1": "DeepSeek-V4.1-Flash (low)",
   "model2": "Claude Sonnet 5.5 (high)",
   "score1": 5,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 107,
   "matchday": 6,
   "model1": "GLM-5.3 (low)",
   "model2": "GNUbg weak",
   "score1": 2,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 388,
   "matchday": 7,
   "model1": "DeepSeek-V4.1-Flash (low)",
   "model2": "GLM-5.3 (low)",
   "score1": 3,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 115,
   "matchday": 7,
   "model1": "GNUbg weak",
   "model2": "Claude Sonnet 5.5 (low)",
   "score1": 3,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 351,
   "matchday": 8,
   "model1": "DeepSeek-V4.1-Flash (low)",
   "model2": "Claude Sonnet 5.5 (low)",
   "score1": 1,
   "score2": 7,
   "status": "finished"
  },
  {
   "games": 8,
   "match_id": 136,
   "matchday": 9,
   "model1": "Gemini 3.8 Flash (high)",
   "model2": "Claude Sonnet 5.5 (high)",
   "score1": 4,
   "score2": 6,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 113,
   "matchday": 9,
   "model1": "DeepSeek-V4.1-Flash (low)",
   "model2": "GNUbg weak",
   "score1": 4,
   "score2": 3,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 326,
   "matchday": 10,
   "model1": "Qwen3.8 Flash (off)",
   "model2": "Gemini 3.8 Flash (high)",
   "score1": 2,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 316,
   "matchday": 10,
   "model1": "Claude Sonnet 5.5 (low)",
   "model2": "Qwen3.8 Flash (off)",
   "score1": 6,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 309,
   "matchday": 10,
   "model1": "Claude Sonnet 5.5 (high)",
   "model2": "Qwen3.8 Flash (off)",
   "score1": 4,
   "score2": 3,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 458,
   "matchday": 10,
   "model1": "Qwen3.8 Flash (off)",
   "model2": "DeepSeek-V4.1-Flash (low)",
   "score1": 3,
   "score2": 6,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 385,
   "matchday": 10,
   "model1": "Qwen3.8 Flash (off)",
   "model2": "GLM-5.3 (low)",
   "score1": 3,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 343,
   "matchday": 10,
   "model1": "Qwen3.8 Flash (off)",
   "model2": "GNUbg weak",
   "score1": 5,
   "score2": 3,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 192,
   "matchday": 10,
   "model1": "Liquid d1-ultra",
   "model2": "GNUbg weak",
   "score1": 5,
   "score2": 6,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 453,
   "matchday": 11,
   "model1": "Claude Fable 5.1 (low)",
   "model2": "GNUbg weak",
   "score1": 3,
   "score2": 6,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 374,
   "matchday": 12,
   "model1": "GNUbg weak",
   "model2": "Fastino GLiDE",
   "score1": 5,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 376,
   "matchday": 12,
   "model1": "Liquid d1-ultra",
   "model2": "Fastino GLiDE",
   "score1": 5,
   "score2": 3,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 201,
   "matchday": 13,
   "model1": "Gemini 3.8 Flash (high)",
   "model2": "Gemma 4 31B (on)",
   "score1": 2,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 213,
   "matchday": 13,
   "model1": "Gemma 4 31B (on)",
   "model2": "Claude Sonnet 5.5 (low)",
   "score1": 3,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 202,
   "matchday": 13,
   "model1": "Gemma 4 31B (on)",
   "model2": "Claude Sonnet 5.5 (high)",
   "score1": 4,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 349,
   "matchday": 13,
   "model1": "Gemma 4 31B (on)",
   "model2": "DeepSeek-V4.1-Flash (low)",
   "score1": 1,
   "score2": 7,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 196,
   "matchday": 13,
   "model1": "Gemma 4 31B (on)",
   "model2": "GLM-5.3 (low)",
   "score1": 4,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 211,
   "matchday": 13,
   "model1": "Gemma 4 31B (on)",
   "model2": "GNUbg weak",
   "score1": 6,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 344,
   "matchday": 13,
   "model1": "Gemma 4 31B (on)",
   "model2": "Qwen3.8 Flash (off)",
   "score1": 7,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 444,
   "matchday": 13,
   "model1": "GNUbg weak",
   "model2": "Grok 4.7 (low)",
   "score1": 2,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 438,
   "matchday": 13,
   "model1": "Claude Fable 5.1 (low)",
   "model2": "Grok 4.7 (low)",
   "score1": 2,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 215,
   "matchday": 14,
   "model1": "Gemini 3.8 Flash (high)",
   "model2": "Gemma 4 31B (off)",
   "score1": 4,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 217,
   "matchday": 14,
   "model1": "Claude Sonnet 5.5 (low)",
   "model2": "Gemma 4 31B (off)",
   "score1": 2,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 216,
   "matchday": 14,
   "model1": "Gemma 4 31B (off)",
   "model2": "Claude Sonnet 5.5 (high)",
   "score1": 2,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 471,
   "matchday": 14,
   "model1": "DeepSeek-V4.1-Flash (low)",
   "model2": "Gemma 4 31B (off)",
   "score1": 5,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 231,
   "matchday": 14,
   "model1": "GLM-5.3 (low)",
   "model2": "Gemma 4 31B (off)",
   "score1": 3,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 227,
   "matchday": 14,
   "model1": "Gemma 4 31B (off)",
   "model2": "GNUbg weak",
   "score1": 2,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 387,
   "matchday": 14,
   "model1": "Qwen3.8 Flash (off)",
   "model2": "Gemma 4 31B (off)",
   "score1": 1,
   "score2": 6,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 230,
   "matchday": 14,
   "model1": "Gemma 4 31B (on)",
   "model2": "Gemma 4 31B (off)",
   "score1": 5,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 565,
   "matchday": 15,
   "model1": "Gemini 3.8 Flash (high)",
   "model2": "Fastino GLiDE",
   "score1": 1,
   "score2": 6,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 567,
   "matchday": 15,
   "model1": "Claude Sonnet 5.5 (low)",
   "model2": "Fastino GLiDE",
   "score1": 5,
   "score2": 3,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 568,
   "matchday": 15,
   "model1": "Claude Sonnet 5.5 (high)",
   "model2": "Fastino GLiDE",
   "score1": 5,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 564,
   "matchday": 15,
   "model1": "Fastino GLiDE",
   "model2": "DeepSeek-V4.1-Flash (low)",
   "score1": 2,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 569,
   "matchday": 15,
   "model1": "Fastino GLiDE",
   "model2": "GLM-5.3 (low)",
   "score1": 3,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 570,
   "matchday": 15,
   "model1": "Fastino GLiDE",
   "model2": "Qwen3.8 Flash (off)",
   "score1": 4,
   "score2": 3,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 571,
   "matchday": 15,
   "model1": "Fastino GLiDE",
   "model2": "Gemma 4 31B (on)",
   "score1": 2,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 572,
   "matchday": 15,
   "model1": "Gemma 4 31B (off)",
   "model2": "Fastino GLiDE",
   "score1": 4,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 574,
   "matchday": 15,
   "model1": "Qwen3.8-27B (low)",
   "model2": "Fastino GLiDE",
   "score1": 6,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 575,
   "matchday": 15,
   "model1": "Claude Fable 5.1 (low)",
   "model2": "Fastino GLiDE",
   "score1": 6,
   "score2": 0,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 222,
   "matchday": 16,
   "model1": "Qwen3.8-27B (low)",
   "model2": "Gemini 3.8 Flash (high)",
   "score1": 1,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 221,
   "matchday": 16,
   "model1": "Qwen3.8-27B (low)",
   "model2": "Claude Sonnet 5.5 (low)",
   "score1": 2,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 205,
   "matchday": 16,
   "model1": "Claude Sonnet 5.5 (high)",
   "model2": "Qwen3.8-27B (low)",
   "score1": 3,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 480,
   "matchday": 16,
   "model1": "DeepSeek-V4.1-Flash (low)",
   "model2": "Qwen3.8-27B (low)",
   "score1": 5,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 367,
   "matchday": 16,
   "model1": "Qwen3.8-27B (low)",
   "model2": "GLM-5.3 (low)",
   "score1": 5,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 337,
   "matchday": 16,
   "model1": "Qwen3.8-27B (low)",
   "model2": "GNUbg weak",
   "score1": 5,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 390,
   "matchday": 16,
   "model1": "Qwen3.8 Flash (off)",
   "model2": "Qwen3.8-27B (low)",
   "score1": 5,
   "score2": 3,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 304,
   "matchday": 16,
   "model1": "Gemma 4 31B (on)",
   "model2": "Qwen3.8-27B (low)",
   "score1": 5,
   "score2": 0,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 340,
   "matchday": 16,
   "model1": "Gemma 4 31B (off)",
   "model2": "Qwen3.8-27B (low)",
   "score1": 2,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 500,
   "matchday": 16,
   "model1": "Gemini 3.8 Flash (high)",
   "model2": "Liquid d1-ultra",
   "score1": 5,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 502,
   "matchday": 16,
   "model1": "Claude Sonnet 5.5 (low)",
   "model2": "Liquid d1-ultra",
   "score1": 5,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 501,
   "matchday": 16,
   "model1": "Liquid d1-ultra",
   "model2": "Claude Sonnet 5.5 (high)",
   "score1": 3,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 517,
   "matchday": 16,
   "model1": "Liquid d1-ultra",
   "model2": "DeepSeek-V4.1-Flash (low)",
   "score1": 2,
   "score2": 8,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 519,
   "matchday": 16,
   "model1": "GLM-5.3 (low)",
   "model2": "Liquid d1-ultra",
   "score1": 5,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 520,
   "matchday": 16,
   "model1": "Qwen3.8 Flash (off)",
   "model2": "Liquid d1-ultra",
   "score1": 2,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 522,
   "matchday": 16,
   "model1": "Liquid d1-ultra",
   "model2": "Gemma 4 31B (on)",
   "score1": 2,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 523,
   "matchday": 16,
   "model1": "Liquid d1-ultra",
   "model2": "Gemma 4 31B (off)",
   "score1": 2,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 525,
   "matchday": 16,
   "model1": "Liquid d1-ultra",
   "model2": "Qwen3.8-27B (low)",
   "score1": 4,
   "score2": 3,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 563,
   "matchday": 16,
   "model1": "Gemini 3.8 Flash (high)",
   "model2": "Grok 4.7 (low)",
   "score1": 5,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 577,
   "matchday": 16,
   "model1": "Grok 4.7 (low)",
   "model2": "Claude Sonnet 5.5 (low)",
   "score1": 2,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 580,
   "matchday": 16,
   "model1": "Grok 4.7 (low)",
   "model2": "Claude Sonnet 5.5 (high)",
   "score1": 7,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 8,
   "match_id": 566,
   "matchday": 16,
   "model1": "Grok 4.7 (low)",
   "model2": "DeepSeek-V4.1-Flash (low)",
   "score1": 5,
   "score2": 3,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 598,
   "matchday": 16,
   "model1": "Grok 4.7 (low)",
   "model2": "GLM-5.3 (low)",
   "score1": 4,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 599,
   "matchday": 16,
   "model1": "Qwen3.8 Flash (off)",
   "model2": "Grok 4.7 (low)",
   "score1": 1,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 620,
   "matchday": 16,
   "model1": "Gemma 4 31B (on)",
   "model2": "Grok 4.7 (low)",
   "score1": 2,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 621,
   "matchday": 16,
   "model1": "Grok 4.7 (low)",
   "model2": "Gemma 4 31B (off)",
   "score1": 5,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 623,
   "matchday": 16,
   "model1": "Grok 4.7 (low)",
   "model2": "Qwen3.8-27B (low)",
   "score1": 3,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 600,
   "matchday": 16,
   "model1": "Grok 4.7 (low)",
   "model2": "Liquid d1-ultra",
   "score1": 3,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 8,
   "match_id": 601,
   "matchday": 16,
   "model1": "Fastino GLiDE",
   "model2": "Grok 4.7 (low)",
   "score1": 3,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 263,
   "matchday": 16,
   "model1": "GNUbg weak",
   "model2": "Jev 1.14.0",
   "score1": 7,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 265,
   "matchday": 16,
   "model1": "Liquid d1-ultra",
   "model2": "Jev 1.14.0",
   "score1": 4,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 377,
   "matchday": 16,
   "model1": "Fastino GLiDE",
   "model2": "Jev 1.14.0",
   "score1": 5,
   "score2": 3,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 506,
   "matchday": 17,
   "model1": "Claude Fable 5.1 (low)",
   "model2": "Gemini 3.8 Flash (high)",
   "score1": 4,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 507,
   "matchday": 17,
   "model1": "Claude Sonnet 5.5 (low)",
   "model2": "Claude Fable 5.1 (low)",
   "score1": 1,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 505,
   "matchday": 17,
   "model1": "Claude Fable 5.1 (low)",
   "model2": "Claude Sonnet 5.5 (high)",
   "score1": 5,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 518,
   "matchday": 17,
   "model1": "Claude Fable 5.1 (low)",
   "model2": "DeepSeek-V4.1-Flash (low)",
   "score1": 1,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 527,
   "matchday": 17,
   "model1": "GLM-5.3 (low)",
   "model2": "Claude Fable 5.1 (low)",
   "score1": 4,
   "score2": 3,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 529,
   "matchday": 17,
   "model1": "Qwen3.8 Flash (off)",
   "model2": "Claude Fable 5.1 (low)",
   "score1": 2,
   "score2": 7,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 551,
   "matchday": 17,
   "model1": "Gemma 4 31B (on)",
   "model2": "Claude Fable 5.1 (low)",
   "score1": 3,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 552,
   "matchday": 17,
   "model1": "Gemma 4 31B (off)",
   "model2": "Claude Fable 5.1 (low)",
   "score1": 6,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 554,
   "matchday": 17,
   "model1": "Claude Fable 5.1 (low)",
   "model2": "Qwen3.8-27B (low)",
   "score1": 4,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 548,
   "matchday": 17,
   "model1": "Liquid d1-ultra",
   "model2": "Claude Fable 5.1 (low)",
   "score1": 1,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 606,
   "matchday": 17,
   "model1": "Gemini 3.8 Flash (high)",
   "model2": "DeepSeek-V4.1-Flash 1017 (off)",
   "score1": 5,
   "score2": 0,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 607,
   "matchday": 17,
   "model1": "Claude Sonnet 5.5 (low)",
   "model2": "DeepSeek-V4.1-Flash 1017 (off)",
   "score1": 5,
   "score2": 3,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 610,
   "matchday": 17,
   "model1": "Claude Sonnet 5.5 (high)",
   "model2": "DeepSeek-V4.1-Flash 1017 (off)",
   "score1": 6,
   "score2": 0,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 626,
   "matchday": 17,
   "model1": "DeepSeek-V4.1-Flash (low)",
   "model2": "DeepSeek-V4.1-Flash 1017 (off)",
   "score1": 2,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 8,
   "match_id": 627,
   "matchday": 17,
   "model1": "DeepSeek-V4.1-Flash 1017 (off)",
   "model2": "GLM-5.3 (low)",
   "score1": 6,
   "score2": 7,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 628,
   "matchday": 17,
   "model1": "DeepSeek-V4.1-Flash 1017 (off)",
   "model2": "GNUbg weak",
   "score1": 6,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 629,
   "matchday": 17,
   "model1": "DeepSeek-V4.1-Flash 1017 (off)",
   "model2": "Qwen3.8 Flash (off)",
   "score1": 6,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 640,
   "matchday": 17,
   "model1": "Gemma 4 31B (on)",
   "model2": "DeepSeek-V4.1-Flash 1017 (off)",
   "score1": 5,
   "score2": 3,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 641,
   "matchday": 17,
   "model1": "DeepSeek-V4.1-Flash 1017 (off)",
   "model2": "Gemma 4 31B (off)",
   "score1": 2,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 639,
   "matchday": 17,
   "model1": "DeepSeek-V4.1-Flash 1017 (off)",
   "model2": "Qwen3.8-27B (low)",
   "score1": 4,
   "score2": 3,
   "status": "finished"
  },
  {
   "games": 8,
   "match_id": 630,
   "matchday": 17,
   "model1": "Liquid d1-ultra",
   "model2": "DeepSeek-V4.1-Flash 1017 (off)",
   "score1": 5,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 631,
   "matchday": 17,
   "model1": "DeepSeek-V4.1-Flash 1017 (off)",
   "model2": "Claude Fable 5.1 (low)",
   "score1": 1,
   "score2": 8,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 632,
   "matchday": 17,
   "model1": "DeepSeek-V4.1-Flash 1017 (off)",
   "model2": "Fastino GLiDE",
   "score1": 4,
   "score2": 6,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 634,
   "matchday": 17,
   "model1": "DeepSeek-V4.1-Flash 1017 (off)",
   "model2": "Grok 4.7 (low)",
   "score1": 1,
   "score2": 6,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 705,
   "matchday": 17,
   "model1": "Clef",
   "model2": "Gemini 3.8 Flash (high)",
   "score1": 2,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 707,
   "matchday": 17,
   "model1": "Clef",
   "model2": "Claude Sonnet 5.5 (low)",
   "score1": 2,
   "score2": 6,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 704,
   "matchday": 17,
   "model1": "Claude Sonnet 5.5 (high)",
   "model2": "Clef",
   "score1": 5,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 709,
   "matchday": 17,
   "model1": "Clef",
   "model2": "DeepSeek-V4.1-Flash (low)",
   "score1": 0,
   "score2": 6,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 715,
   "matchday": 17,
   "model1": "GLM-5.3 (low)",
   "model2": "Clef",
   "score1": 2,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 78,
   "matchday": 17,
   "model1": "Clef",
   "model2": "GNUbg weak",
   "score1": 2,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 716,
   "matchday": 17,
   "model1": "Qwen3.8 Flash (off)",
   "model2": "Clef",
   "score1": 4,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 717,
   "matchday": 17,
   "model1": "Clef",
   "model2": "Gemma 4 31B (on)",
   "score1": 4,
   "score2": 6,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 718,
   "matchday": 17,
   "model1": "Clef",
   "model2": "Gemma 4 31B (off)",
   "score1": 0,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 719,
   "matchday": 17,
   "model1": "Qwen3.8-27B (low)",
   "model2": "Clef",
   "score1": 5,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 180,
   "matchday": 17,
   "model1": "Clef",
   "model2": "Liquid d1-ultra",
   "score1": 1,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 721,
   "matchday": 17,
   "model1": "Claude Fable 5.1 (low)",
   "model2": "Clef",
   "score1": 7,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 359,
   "matchday": 17,
   "model1": "Fastino GLiDE",
   "model2": "Clef",
   "score1": 6,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 722,
   "matchday": 17,
   "model1": "Grok 4.7 (low)",
   "model2": "Clef",
   "score1": 7,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 723,
   "matchday": 17,
   "model1": "Clef",
   "model2": "DeepSeek-V4.1-Flash 1017 (off)",
   "score1": 4,
   "score2": 6,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 724,
   "matchday": 17,
   "model1": "Clef",
   "model2": "DeepSeek-V4.1-Flash 1017 (low)",
   "score1": 0,
   "score2": 8,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 245,
   "matchday": 17,
   "model1": "Clef",
   "model2": "Jev 1.14.0",
   "score1": 3,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 8,
   "match_id": 609,
   "matchday": 18,
   "model1": "DeepSeek-V4.1-Flash 1017 (low)",
   "model2": "Gemini 3.8 Flash (high)",
   "score1": 5,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 8,
   "match_id": 613,
   "matchday": 18,
   "model1": "Claude Sonnet 5.5 (low)",
   "model2": "DeepSeek-V4.1-Flash 1017 (low)",
   "score1": 7,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 612,
   "matchday": 18,
   "model1": "DeepSeek-V4.1-Flash 1017 (low)",
   "model2": "Claude Sonnet 5.5 (high)",
   "score1": 2,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 635,
   "matchday": 18,
   "model1": "DeepSeek-V4.1-Flash 1017 (low)",
   "model2": "DeepSeek-V4.1-Flash (low)",
   "score1": 6,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 636,
   "matchday": 18,
   "model1": "DeepSeek-V4.1-Flash 1017 (low)",
   "model2": "GLM-5.3 (low)",
   "score1": 5,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 637,
   "matchday": 18,
   "model1": "DeepSeek-V4.1-Flash 1017 (low)",
   "model2": "GNUbg weak",
   "score1": 7,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 642,
   "matchday": 18,
   "model1": "Qwen3.8 Flash (off)",
   "model2": "DeepSeek-V4.1-Flash 1017 (low)",
   "score1": 2,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 649,
   "matchday": 18,
   "model1": "DeepSeek-V4.1-Flash 1017 (low)",
   "model2": "Gemma 4 31B (on)",
   "score1": 7,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 650,
   "matchday": 18,
   "model1": "Gemma 4 31B (off)",
   "model2": "DeepSeek-V4.1-Flash 1017 (low)",
   "score1": 4,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 648,
   "matchday": 18,
   "model1": "DeepSeek-V4.1-Flash 1017 (low)",
   "model2": "Qwen3.8-27B (low)",
   "score1": 2,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 643,
   "matchday": 18,
   "model1": "DeepSeek-V4.1-Flash 1017 (low)",
   "model2": "Liquid d1-ultra",
   "score1": 5,
   "score2": 3,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 644,
   "matchday": 18,
   "model1": "Claude Fable 5.1 (low)",
   "model2": "DeepSeek-V4.1-Flash 1017 (low)",
   "score1": 1,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 645,
   "matchday": 18,
   "model1": "Fastino GLiDE",
   "model2": "DeepSeek-V4.1-Flash 1017 (low)",
   "score1": 2,
   "score2": 7,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 646,
   "matchday": 18,
   "model1": "DeepSeek-V4.1-Flash 1017 (low)",
   "model2": "Grok 4.7 (low)",
   "score1": 4,
   "score2": 3,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 647,
   "matchday": 18,
   "model1": "DeepSeek-V4.1-Flash 1017 (off)",
   "model2": "DeepSeek-V4.1-Flash 1017 (low)",
   "score1": 1,
   "score2": 7,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 672,
   "matchday": 18,
   "model1": "Jev 1.14.0",
   "model2": "Gemini 3.8 Flash (high)",
   "score1": 1,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 667,
   "matchday": 18,
   "model1": "Claude Sonnet 5.5 (low)",
   "model2": "Jev 1.14.0",
   "score1": 7,
   "score2": 0,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 674,
   "matchday": 18,
   "model1": "Jev 1.14.0",
   "model2": "Claude Sonnet 5.5 (high)",
   "score1": 0,
   "score2": 6,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 675,
   "matchday": 18,
   "model1": "DeepSeek-V4.1-Flash (low)",
   "model2": "Jev 1.14.0",
   "score1": 7,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 676,
   "matchday": 18,
   "model1": "GLM-5.3 (low)",
   "model2": "Jev 1.14.0",
   "score1": 2,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 679,
   "matchday": 18,
   "model1": "Qwen3.8 Flash (off)",
   "model2": "Jev 1.14.0",
   "score1": 1,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 696,
   "matchday": 18,
   "model1": "Jev 1.14.0",
   "model2": "Gemma 4 31B (on)",
   "score1": 3,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 697,
   "matchday": 18,
   "model1": "Gemma 4 31B (off)",
   "model2": "Jev 1.14.0",
   "score1": 5,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 698,
   "matchday": 18,
   "model1": "Jev 1.14.0",
   "model2": "Qwen3.8-27B (low)",
   "score1": 1,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 700,
   "matchday": 18,
   "model1": "Jev 1.14.0",
   "model2": "Claude Fable 5.1 (low)",
   "score1": 0,
   "score2": 6,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 701,
   "matchday": 18,
   "model1": "Grok 4.7 (low)",
   "model2": "Jev 1.14.0",
   "score1": 5,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 702,
   "matchday": 18,
   "model1": "DeepSeek-V4.1-Flash 1017 (off)",
   "model2": "Jev 1.14.0",
   "score1": 0,
   "score2": 6,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 703,
   "matchday": 18,
   "model1": "DeepSeek-V4.1-Flash 1017 (low)",
   "model2": "Jev 1.14.0",
   "score1": 3,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 137,
   "matchday": 18,
   "model1": "Gemini 3.8 Flash (low)",
   "model2": "Gemini 3.8 Flash (high)",
   "score1": 2,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 8,
   "match_id": 128,
   "matchday": 18,
   "model1": "Claude Sonnet 5.5 (low)",
   "model2": "Gemini 3.8 Flash (low)",
   "score1": 4,
   "score2": 4,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 123,
   "matchday": 18,
   "model1": "Claude Sonnet 5.5 (high)",
   "model2": "Gemini 3.8 Flash (low)",
   "score1": 2,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 120,
   "matchday": 18,
   "model1": "Gemini 3.8 Flash (low)",
   "model2": "DeepSeek-V4.1-Flash (low)",
   "score1": 6,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 125,
   "matchday": 18,
   "model1": "GLM-5.3 (low)",
   "model2": "Gemini 3.8 Flash (low)",
   "score1": 1,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 129,
   "matchday": 18,
   "model1": "Gemini 3.8 Flash (low)",
   "model2": "GNUbg weak",
   "score1": 8,
   "score2": 0,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 315,
   "matchday": 18,
   "model1": "Qwen3.8 Flash (off)",
   "model2": "Gemini 3.8 Flash (low)",
   "score1": 1,
   "score2": 6,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 214,
   "matchday": 18,
   "model1": "Gemma 4 31B (on)",
   "model2": "Gemini 3.8 Flash (low)",
   "score1": 5,
   "score2": 1,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 203,
   "matchday": 18,
   "model1": "Gemini 3.8 Flash (low)",
   "model2": "Gemma 4 31B (off)",
   "score1": 7,
   "score2": 3,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 220,
   "matchday": 18,
   "model1": "Qwen3.8-27B (low)",
   "model2": "Gemini 3.8 Flash (low)",
   "score1": 3,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 503,
   "matchday": 18,
   "model1": "Gemini 3.8 Flash (low)",
   "model2": "Liquid d1-ultra",
   "score1": 5,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 504,
   "matchday": 18,
   "model1": "Claude Fable 5.1 (low)",
   "model2": "Gemini 3.8 Flash (low)",
   "score1": 1,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 560,
   "matchday": 18,
   "model1": "Fastino GLiDE",
   "model2": "Gemini 3.8 Flash (low)",
   "score1": 3,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 559,
   "matchday": 18,
   "model1": "Gemini 3.8 Flash (low)",
   "model2": "Grok 4.7 (low)",
   "score1": 5,
   "score2": 2,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 608,
   "matchday": 18,
   "model1": "Gemini 3.8 Flash (low)",
   "model2": "DeepSeek-V4.1-Flash 1017 (off)",
   "score1": 5,
   "score2": 3,
   "status": "finished"
  },
  {
   "games": 8,
   "match_id": 611,
   "matchday": 18,
   "model1": "Gemini 3.8 Flash (low)",
   "model2": "DeepSeek-V4.1-Flash 1017 (low)",
   "score1": 5,
   "score2": 5,
   "status": "finished"
  },
  {
   "games": 4,
   "match_id": 711,
   "matchday": 18,
   "model1": "Gemini 3.8 Flash (low)",
   "model2": "Jev 1.14.0",
   "score1": 5,
   "score2": 0,
   "status": "finished"
  },
  {
   "games": 6,
   "match_id": 727,
   "matchday": 18,
   "model1": "Clef",
   "model2": "Gemini 3.8 Flash (low)",
   "score1": 2,
   "score2": 5,
   "status": "finished"
  }
 ],
 "metric_status": "Published",
 "offline": {
  "blocks": 477,
  "calibration": {
   "Clef": {
    "chosen": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.08369473928556728,
       "n": 4,
       "observed": 0.0
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.15700029144169536,
       "n": 34,
       "observed": 0.20588235294117646
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.25157560131247875,
       "n": 91,
       "observed": 0.27472527472527475
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.3449918499653396,
       "n": 99,
       "observed": 0.3838383838383838
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.45191892779483556,
       "n": 85,
       "observed": 0.4
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5475372909817964,
       "n": 70,
       "observed": 0.38571428571428573
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6501698073934983,
       "n": 60,
       "observed": 0.7333333333333333
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7452897699399292,
       "n": 37,
       "observed": 0.8378378378378378
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.850096512922872,
       "n": 27,
       "observed": 0.8518518518518519
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9139395041590126,
       "n": 2,
       "observed": 1.0
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.072696875,
       "n": 32,
       "observed": 0.1875
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.14848295454545457,
       "n": 176,
       "observed": 0.32386363636363635
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.24917333333333333,
       "n": 120,
       "observed": 0.375
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.34431999999999996,
       "n": 75,
       "observed": 0.4666666666666667
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.4393333333333333,
       "n": 51,
       "observed": 0.8235294117647058
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5364107142857143,
       "n": 28,
       "observed": 0.8214285714285714
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6490157894736841,
       "n": 19,
       "observed": 0.7894736842105263
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7308375,
       "n": 8,
       "observed": 1.0
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": null,
       "n": 0,
       "observed": null
      }
     ],
     "brier_after": 0.21090271019140494,
     "brier_before": 0.24305036275049116,
     "event_rate": 0.4538310412573674,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 509,
     "records": 509,
     "too_few": false
    },
    "risk_class": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.0606046228268364,
       "n": 17,
       "observed": 0.29411764705882354
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.14952091426341824,
       "n": 27,
       "observed": 0.18518518518518517
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.254070658565149,
       "n": 51,
       "observed": 0.1568627450980392
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.35315626036946246,
       "n": 69,
       "observed": 0.3333333333333333
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.45322446483660694,
       "n": 63,
       "observed": 0.31746031746031744
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5546670925497703,
       "n": 51,
       "observed": 0.6862745098039216
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6447885632935857,
       "n": 47,
       "observed": 0.7021276595744681
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7402422818459427,
       "n": 38,
       "observed": 0.7368421052631579
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8263042188032229,
       "n": 18,
       "observed": 0.9444444444444444
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9486276295038966,
       "n": 1,
       "observed": 0.0
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.0922,
       "n": 1,
       "observed": 0.0
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.171423198319832,
       "n": 5,
       "observed": 0.6
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.2648784095554786,
       "n": 15,
       "observed": 0.13333333333333333
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.35145005628930803,
       "n": 20,
       "observed": 0.25
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.46367274041691253,
       "n": 24,
       "observed": 0.125
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5549126132210778,
       "n": 45,
       "observed": 0.13333333333333333
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6467533768261168,
       "n": 61,
       "observed": 0.36065573770491804
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.746297505627637,
       "n": 70,
       "observed": 0.42857142857142855
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8495931050205263,
       "n": 89,
       "observed": 0.6853932584269663
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.92807389159653,
       "n": 52,
       "observed": 0.8076923076923077
      }
     ],
     "brier_after": 0.20405808764046265,
     "brier_before": 0.26221422152594365,
     "event_rate": 0.45549738219895286,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 382,
     "records": 382,
     "too_few": false
    }
   },
   "DeepSeek-V4.1-Flash (low)": {
    "chosen": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7742880278835842,
       "n": 297,
       "observed": 0.835016835016835
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8134847318636248,
       "n": 111,
       "observed": 0.6936936936936937
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": null,
       "n": 0,
       "observed": null
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9999118874889621,
       "n": 408,
       "observed": 0.7965686274509803
      }
     ],
     "brier_after": 0.16576315660501362,
     "brier_before": 0.20339778569124506,
     "event_rate": 0.7965686274509803,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 408,
     "records": 408,
     "too_few": false
    },
    "risk_class": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.17992922242565307,
       "n": 100,
       "observed": 0.19
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.23238722773176976,
       "n": 73,
       "observed": 0.2054794520547945
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.3093320646835974,
       "n": 1,
       "observed": 0.0
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5656870265064462,
       "n": 3,
       "observed": 0.6666666666666666
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6426951842305085,
       "n": 21,
       "observed": 0.7142857142857143
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.714259763198103,
       "n": 6,
       "observed": 0.5
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": null,
       "n": 0,
       "observed": null
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 7.1376747111484e-05,
       "n": 174,
       "observed": 0.19540229885057472
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9999096920648053,
       "n": 30,
       "observed": 0.6666666666666666
      }
     ],
     "brier_after": 0.16916866961085808,
     "brier_before": 0.21564400853444152,
     "event_rate": 0.2647058823529412,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 204,
     "records": 204,
     "too_few": false
    }
   },
   "DeepSeek-V4.1-Flash 1017 (low)": {
    "chosen": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.4895781902292056,
       "n": 1,
       "observed": 1.0
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6965900023442233,
       "n": 1,
       "observed": 1.0
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7716583387430861,
       "n": 5,
       "observed": 1.0
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8357999000261165,
       "n": 166,
       "observed": 0.8373493975903614
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9769530383699566,
       "n": 7,
       "observed": 0.5714285714285714
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9999988571238737,
       "n": 180,
       "observed": 0.8333333333333334
      }
     ],
     "brier_after": 0.14803488413712623,
     "brier_before": 0.16666626201824844,
     "event_rate": 0.8333333333333334,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 180,
     "records": 180,
     "too_few": true
    },
    "risk_class": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.18940210639379718,
       "n": 23,
       "observed": 0.2608695652173913
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.22356577054213775,
       "n": 48,
       "observed": 0.1875
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6695315495732789,
       "n": 3,
       "observed": 1.0
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7475977235247153,
       "n": 1,
       "observed": 1.0
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8528791897222666,
       "n": 5,
       "observed": 0.6
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": null,
       "n": 0,
       "observed": null
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 6.506753164204642e-07,
       "n": 71,
       "observed": 0.2112676056338028
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.999999297282229,
       "n": 9,
       "observed": 0.7777777777777778
      }
     ],
     "brier_after": 0.17399427634949047,
     "brier_before": 0.21249963441251266,
     "event_rate": 0.275,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 80,
     "records": 80,
     "too_few": true
    }
   },
   "DeepSeek-V4.1-Flash 1017 (off)": {
    "chosen": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.3952219585310166,
       "n": 1,
       "observed": 0.0
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.4669666451155729,
       "n": 86,
       "observed": 0.4418604651162791
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5455906324198114,
       "n": 106,
       "observed": 0.6132075471698113
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.647407403170968,
       "n": 82,
       "observed": 0.6463414634146342
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7472121728413627,
       "n": 119,
       "observed": 0.7142857142857143
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8428888297628158,
       "n": 132,
       "observed": 0.8333333333333334
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9067749432987559,
       "n": 11,
       "observed": 0.9090909090909091
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.3710487757738571,
       "n": 5,
       "observed": 0.6
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.4649396352893289,
       "n": 6,
       "observed": 0.3333333333333333
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5544078521761625,
       "n": 37,
       "observed": 0.2972972972972973
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6500232378755822,
       "n": 29,
       "observed": 0.5517241379310345
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7580950143023419,
       "n": 43,
       "observed": 0.4883720930232558
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8503803459957399,
       "n": 47,
       "observed": 0.6382978723404256
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9857935875026801,
       "n": 370,
       "observed": 0.7513513513513513
      }
     ],
     "brier_after": 0.20186752561245486,
     "brier_before": 0.2543665112863535,
     "event_rate": 0.6722532588454376,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 537,
     "records": 537,
     "too_few": false
    },
    "risk_class": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.0937273246437668,
       "n": 2,
       "observed": 0.0
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.1566708685536635,
       "n": 75,
       "observed": 0.17333333333333334
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.24592714999002946,
       "n": 79,
       "observed": 0.24050632911392406
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.3479550817054814,
       "n": 47,
       "observed": 0.3191489361702128
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.44730051383170555,
       "n": 20,
       "observed": 0.7
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5278183411797372,
       "n": 11,
       "observed": 0.36363636363636365
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6384349873688168,
       "n": 5,
       "observed": 0.4
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7619572186669168,
       "n": 3,
       "observed": 1.0
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8332292753419092,
       "n": 4,
       "observed": 0.5
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": null,
       "n": 0,
       "observed": null
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.01160791620315511,
       "n": 194,
       "observed": 0.22164948453608246
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.14237133206012148,
       "n": 10,
       "observed": 0.3
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.23305635869878744,
       "n": 8,
       "observed": 0.875
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.3390759984872154,
       "n": 2,
       "observed": 0.5
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.4507154921542657,
       "n": 6,
       "observed": 0.8333333333333334
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.568123282322499,
       "n": 3,
       "observed": 0.6666666666666666
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6320629171539349,
       "n": 2,
       "observed": 0.5
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7544937376250908,
       "n": 4,
       "observed": 0.25
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8499949950310292,
       "n": 5,
       "observed": 0.4
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9822325051729884,
       "n": 12,
       "observed": 0.5833333333333334
      }
     ],
     "brier_after": 0.18932917949338296,
     "brier_before": 0.2452913432880394,
     "event_rate": 0.2926829268292683,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 246,
     "records": 246,
     "too_few": false
    }
   },
   "Fastino GLiDE": {
    "chosen": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.08212769068713499,
       "n": 7,
       "observed": 0.0
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.16546154425825563,
       "n": 95,
       "observed": 0.2
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.2565509915164336,
       "n": 255,
       "observed": 0.2823529411764706
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.34695161317112744,
       "n": 356,
       "observed": 0.34269662921348315
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.44981072543456047,
       "n": 369,
       "observed": 0.4254742547425474
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5505936433414326,
       "n": 322,
       "observed": 0.5403726708074534
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6446798497254661,
       "n": 239,
       "observed": 0.6736401673640168
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.741557993295663,
       "n": 172,
       "observed": 0.7151162790697675
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8455839326234179,
       "n": 111,
       "observed": 0.8828828828828829
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9381851436115716,
       "n": 21,
       "observed": 0.9047619047619048
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.07750323546786204,
       "n": 31,
       "observed": 0.03225806451612903
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.1583210187128111,
       "n": 193,
       "observed": 0.26424870466321243
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.2514122658544838,
       "n": 357,
       "observed": 0.29411764705882354
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.3523590681994324,
       "n": 321,
       "observed": 0.4174454828660436
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.44722482875437,
       "n": 297,
       "observed": 0.4781144781144781
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5478436280570192,
       "n": 277,
       "observed": 0.5523465703971119
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6473546342940474,
       "n": 191,
       "observed": 0.6963350785340314
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.743749504632671,
       "n": 146,
       "observed": 0.726027397260274
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8487239155643034,
       "n": 105,
       "observed": 0.8952380952380953
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9372048353095866,
       "n": 29,
       "observed": 0.896551724137931
      }
     ],
     "brier_after": 0.21443719144808535,
     "brier_before": 0.21594826639184334,
     "event_rate": 0.48536209553158705,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 1947,
     "records": 1947,
     "too_few": false
    },
    "risk_class": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.06579686668471407,
       "n": 83,
       "observed": 0.07228915662650602
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.15046631801927404,
       "n": 191,
       "observed": 0.13612565445026178
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.24821938295147455,
       "n": 200,
       "observed": 0.27
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.3511102942504446,
       "n": 216,
       "observed": 0.3425925925925926
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.44852290045395543,
       "n": 187,
       "observed": 0.5026737967914439
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5489564206986696,
       "n": 154,
       "observed": 0.525974025974026
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6501796656651213,
       "n": 143,
       "observed": 0.5874125874125874
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7478442383808622,
       "n": 128,
       "observed": 0.71875
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8468367501106433,
       "n": 105,
       "observed": 0.8952380952380953
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9175404345293884,
       "n": 19,
       "observed": 0.9473684210526315
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.06096705715813477,
       "n": 74,
       "observed": 0.06756756756756757
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.15100584900960848,
       "n": 145,
       "observed": 0.1103448275862069
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.25168310836016056,
       "n": 150,
       "observed": 0.25333333333333335
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.3517394468086782,
       "n": 139,
       "observed": 0.26618705035971224
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.45055419972111194,
       "n": 168,
       "observed": 0.35714285714285715
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5496229286717574,
       "n": 157,
       "observed": 0.45222929936305734
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6465639950462744,
       "n": 145,
       "observed": 0.5517241379310345
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7486031482636042,
       "n": 159,
       "observed": 0.5534591194968553
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8504742068821075,
       "n": 169,
       "observed": 0.7041420118343196
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9355845931606093,
       "n": 120,
       "observed": 0.9083333333333333
      }
     ],
     "brier_after": 0.19142955638354162,
     "brier_before": 0.2005046451028868,
     "event_rate": 0.43688639551192143,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 1426,
     "records": 1426,
     "too_few": false
    }
   },
   "GLM-5.3 (low)": {
    "chosen": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.48944831224180124,
       "n": 9,
       "observed": 0.2222222222222222
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5535246353871159,
       "n": 78,
       "observed": 0.6410256410256411
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6594223760177421,
       "n": 229,
       "observed": 0.6724890829694323
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7385159282258463,
       "n": 198,
       "observed": 0.696969696969697
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.817753266175335,
       "n": 13,
       "observed": 0.9230769230769231
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": null,
       "n": 0,
       "observed": null
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.3611824762582574,
       "n": 2,
       "observed": 0.0
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.46566567178344836,
       "n": 9,
       "observed": 0.3333333333333333
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5516671249600849,
       "n": 15,
       "observed": 0.4666666666666667
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6517781308792289,
       "n": 12,
       "observed": 0.3333333333333333
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7482397669205202,
       "n": 24,
       "observed": 0.7083333333333334
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8650218401291577,
       "n": 35,
       "observed": 0.7428571428571429
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9796583086035173,
       "n": 430,
       "observed": 0.6953488372093023
      }
     ],
     "brier_after": 0.21497005183283677,
     "brier_before": 0.2820497653624259,
     "event_rate": 0.6755218216318786,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 527,
     "records": 527,
     "too_few": false
    },
    "risk_class": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.18352752776755146,
       "n": 28,
       "observed": 0.21428571428571427
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.25375935776692704,
       "n": 129,
       "observed": 0.29457364341085274
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.3310492653424282,
       "n": 61,
       "observed": 0.21311475409836064
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.4543855221527018,
       "n": 18,
       "observed": 0.5
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5419550386112114,
       "n": 6,
       "observed": 0.6666666666666666
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6260343003003197,
       "n": 11,
       "observed": 0.45454545454545453
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": null,
       "n": 0,
       "observed": null
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.020712978411246415,
       "n": 188,
       "observed": 0.25
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.13641950930189878,
       "n": 19,
       "observed": 0.3684210526315789
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.2526008436912488,
       "n": 9,
       "observed": 0.2222222222222222
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.3441397374625125,
       "n": 2,
       "observed": 0.0
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5671570862856203,
       "n": 5,
       "observed": 0.6
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.643555083780795,
       "n": 3,
       "observed": 0.0
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7520091788212007,
       "n": 3,
       "observed": 0.6666666666666666
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8444326803627741,
       "n": 3,
       "observed": 0.6666666666666666
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9848667007017321,
       "n": 21,
       "observed": 0.5714285714285714
      }
     ],
     "brier_after": 0.20542511971660465,
     "brier_before": 0.2568880121560243,
     "event_rate": 0.2964426877470356,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 253,
     "records": 253,
     "too_few": false
    }
   },
   "Gemma 4 31B (off)": {
    "chosen": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.35710703579146447,
       "n": 29,
       "observed": 0.5517241379310345
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.45122675119465433,
       "n": 68,
       "observed": 0.47058823529411764
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5492013573795403,
       "n": 57,
       "observed": 0.5263157894736842
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6467961359476133,
       "n": 89,
       "observed": 0.5842696629213483
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7513324039302757,
       "n": 100,
       "observed": 0.69
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8392267758290923,
       "n": 132,
       "observed": 0.8863636363636364
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": null,
       "n": 0,
       "observed": null
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.42234082470473394,
       "n": 1,
       "observed": 1.0
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5516732192633077,
       "n": 9,
       "observed": 0.5555555555555556
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6568512956363473,
       "n": 10,
       "observed": 0.6
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7521751650678901,
       "n": 10,
       "observed": 0.6
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8513030823804353,
       "n": 20,
       "observed": 0.4
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9922268175431854,
       "n": 425,
       "observed": 0.6823529411764706
      }
     ],
     "brier_after": 0.2026266775779046,
     "brier_before": 0.31121211169473384,
     "event_rate": 0.6652631578947369,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 475,
     "records": 475,
     "too_few": false
    },
    "risk_class": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.17448068485160953,
       "n": 73,
       "observed": 0.0821917808219178
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.2463727297974186,
       "n": 61,
       "observed": 0.3114754098360656
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.3435543539265975,
       "n": 26,
       "observed": 0.5384615384615384
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.4265911246594425,
       "n": 10,
       "observed": 0.4
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5464960063195213,
       "n": 11,
       "observed": 0.5454545454545454
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6559107091307248,
       "n": 11,
       "observed": 0.6363636363636364
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7617134212936104,
       "n": 11,
       "observed": 0.9090909090909091
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8505488699779905,
       "n": 18,
       "observed": 0.7222222222222222
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9229973544468149,
       "n": 4,
       "observed": 0.5
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.0016799185877899878,
       "n": 172,
       "observed": 0.2558139534883721
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.17985927328881748,
       "n": 3,
       "observed": 1.0
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.23376720429867104,
       "n": 2,
       "observed": 0.5
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.385726322198231,
       "n": 2,
       "observed": 0.0
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5643500303873065,
       "n": 3,
       "observed": 0.6666666666666666
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6226416178079421,
       "n": 1,
       "observed": 0.0
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.841918739917527,
       "n": 3,
       "observed": 0.6666666666666666
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9898410116791285,
       "n": 39,
       "observed": 0.7435897435897436
      }
     ],
     "brier_after": 0.18526302731493774,
     "brier_before": 0.258921230005358,
     "event_rate": 0.36,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 225,
     "records": 225,
     "too_few": false
    }
   },
   "Gemma 4 31B (on)": {
    "chosen": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.0831670467688569,
       "n": 1,
       "observed": 1.0
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.26625113206390316,
       "n": 1,
       "observed": 0.0
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.4304701228836291,
       "n": 1,
       "observed": 0.0
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5357731675854533,
       "n": 2,
       "observed": 0.5
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6502010223909183,
       "n": 2,
       "observed": 0.0
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7500212089207765,
       "n": 6,
       "observed": 0.8333333333333334
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8631654184327809,
       "n": 156,
       "observed": 0.8653846153846154
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": null,
       "n": 0,
       "observed": null
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9999734228884493,
       "n": 169,
       "observed": 0.8402366863905325
      }
     ],
     "brier_after": 0.1269123650521065,
     "brier_before": 0.15973414209478073,
     "event_rate": 0.8402366863905325,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 169,
     "records": 169,
     "too_few": true
    },
    "risk_class": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.1493385198028323,
       "n": 65,
       "observed": 0.15384615384615385
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.2557692045586026,
       "n": 2,
       "observed": 0.5
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6245221724714133,
       "n": 1,
       "observed": 0.0
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9856322286065082,
       "n": 3,
       "observed": 1.0
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 4.259964312091741e-07,
       "n": 68,
       "observed": 0.16176470588235295
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.999432969016007,
       "n": 3,
       "observed": 1.0
      }
     ],
     "brier_after": 0.13552562727814538,
     "brier_before": 0.15492922130003695,
     "event_rate": 0.19718309859154928,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 71,
     "records": 71,
     "too_few": true
    }
   },
   "Jev 1.14.0": {
    "chosen": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.0919936720451845,
       "n": 1,
       "observed": 0.0
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.16211552593258904,
       "n": 25,
       "observed": 0.24
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.2498935871827937,
       "n": 79,
       "observed": 0.2911392405063291
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.34440374520513695,
       "n": 100,
       "observed": 0.34
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.4448401379603752,
       "n": 87,
       "observed": 0.39080459770114945
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5507168019772875,
       "n": 60,
       "observed": 0.5166666666666667
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6538349978413841,
       "n": 62,
       "observed": 0.6290322580645161
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7469063113512968,
       "n": 47,
       "observed": 0.7872340425531915
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8426770302064616,
       "n": 41,
       "observed": 0.8780487804878049
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9420698478885094,
       "n": 5,
       "observed": 1.0
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.16899999999999998,
       "n": 10,
       "observed": 0.2
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.25574074074074077,
       "n": 54,
       "observed": 0.24074074074074073
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.3487209302325582,
       "n": 86,
       "observed": 0.3372093023255814
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.4429166666666668,
       "n": 96,
       "observed": 0.34375
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5413698630136986,
       "n": 73,
       "observed": 0.4657534246575342
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6433333333333333,
       "n": 60,
       "observed": 0.5666666666666667
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.740735294117647,
       "n": 68,
       "observed": 0.6911764705882353
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8461224489795919,
       "n": 49,
       "observed": 0.8979591836734694
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9227272727272726,
       "n": 11,
       "observed": 0.8181818181818182
      }
     ],
     "brier_after": 0.21025126767318267,
     "brier_before": 0.21103057199211048,
     "event_rate": 0.4832347140039448,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 507,
     "records": 507,
     "too_few": false
    },
    "risk_class": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.07251724808274812,
       "n": 13,
       "observed": 0.07692307692307693
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.1562266310249055,
       "n": 37,
       "observed": 0.24324324324324326
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.24869429882954008,
       "n": 51,
       "observed": 0.27450980392156865
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.3543162283155718,
       "n": 50,
       "observed": 0.3
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.4403452247415148,
       "n": 50,
       "observed": 0.36
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5492935685008772,
       "n": 46,
       "observed": 0.5869565217391305
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6487096894508647,
       "n": 47,
       "observed": 0.6808510638297872
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7551350160415679,
       "n": 31,
       "observed": 0.6451612903225806
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8495405498821984,
       "n": 25,
       "observed": 0.88
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9364198055738728,
       "n": 17,
       "observed": 1.0
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.05768106597893833,
       "n": 47,
       "observed": 0.1702127659574468
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.14267772060224893,
       "n": 53,
       "observed": 0.2830188679245283
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.24872929292929297,
       "n": 50,
       "observed": 0.28
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.3356974506974507,
       "n": 42,
       "observed": 0.42857142857142855
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.4388139459107201,
       "n": 31,
       "observed": 0.45161290322580644
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.541350086228135,
       "n": 41,
       "observed": 0.5853658536585366
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6403282828282828,
       "n": 28,
       "observed": 0.7142857142857143
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7426551226551227,
       "n": 28,
       "observed": 0.6785714285714286
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8470244107744108,
       "n": 24,
       "observed": 0.8333333333333334
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9464602547211243,
       "n": 23,
       "observed": 1.0
      }
     ],
     "brier_after": 0.19917975499019736,
     "brier_before": 0.19990484441475276,
     "event_rate": 0.4768392370572207,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 367,
     "records": 367,
     "too_few": false
    }
   },
   "Liquid d1-ultra": {
    "chosen": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.09120055918019579,
       "n": 5,
       "observed": 0.4
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.16465638226031493,
       "n": 106,
       "observed": 0.16037735849056603
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.2565937260742635,
       "n": 348,
       "observed": 0.22988505747126436
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.34963028416418945,
       "n": 407,
       "observed": 0.35626535626535627
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.4506276513864841,
       "n": 307,
       "observed": 0.4723127035830619
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5472089369815808,
       "n": 277,
       "observed": 0.5270758122743683
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6501001523185911,
       "n": 226,
       "observed": 0.6814159292035398
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7489515904770628,
       "n": 185,
       "observed": 0.7621621621621621
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8471712541907055,
       "n": 139,
       "observed": 0.8129496402877698
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9328227476082744,
       "n": 70,
       "observed": 0.9
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.08448914761946508,
       "n": 15,
       "observed": 0.13333333333333333
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.16384920332914868,
       "n": 277,
       "observed": 0.18411552346570398
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.2488701448767924,
       "n": 535,
       "observed": 0.32897196261682243
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.3490973678207701,
       "n": 381,
       "observed": 0.45931758530183725
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.4465303263262194,
       "n": 287,
       "observed": 0.5505226480836237
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5491666768328624,
       "n": 217,
       "observed": 0.7004608294930875
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.649754128180859,
       "n": 161,
       "observed": 0.7763975155279503
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7485384998807756,
       "n": 109,
       "observed": 0.8256880733944955
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.845026996144843,
       "n": 68,
       "observed": 0.8529411764705882
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9287893823349282,
       "n": 20,
       "observed": 0.95
      }
     ],
     "brier_after": 0.20535780944714443,
     "brier_before": 0.21372114742457327,
     "event_rate": 0.4859903381642512,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 2070,
     "records": 2070,
     "too_few": false
    },
    "risk_class": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.06578555994934429,
       "n": 78,
       "observed": 0.14102564102564102
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.14723376657117898,
       "n": 190,
       "observed": 0.15789473684210525
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.24831265961847357,
       "n": 189,
       "observed": 0.2275132275132275
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.3503590701753975,
       "n": 203,
       "observed": 0.3054187192118227
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.4479542179437588,
       "n": 192,
       "observed": 0.4322916666666667
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5493230237772172,
       "n": 187,
       "observed": 0.5401069518716578
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6493544786186947,
       "n": 180,
       "observed": 0.6833333333333333
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7461934657810022,
       "n": 130,
       "observed": 0.7384615384615385
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8469539423925276,
       "n": 103,
       "observed": 0.8737864077669902
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9348807529511507,
       "n": 51,
       "observed": 0.9411764705882353
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.06659228795344718,
       "n": 57,
       "observed": 0.10526315789473684
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.1505919194323934,
       "n": 139,
       "observed": 0.17985611510791366
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.2516385372330242,
       "n": 144,
       "observed": 0.1597222222222222
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.35096570927121507,
       "n": 141,
       "observed": 0.24822695035460993
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.4501239514444574,
       "n": 158,
       "observed": 0.31645569620253167
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5460459934463894,
       "n": 168,
       "observed": 0.4226190476190476
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6478530248470159,
       "n": 176,
       "observed": 0.5397727272727273
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7492405279981862,
       "n": 208,
       "observed": 0.6057692307692307
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8473844704608967,
       "n": 174,
       "observed": 0.7471264367816092
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.942326441156748,
       "n": 138,
       "observed": 0.9130434782608695
      }
     ],
     "brier_after": 0.18973004327747733,
     "brier_before": 0.19897989014720355,
     "event_rate": 0.45708582834331335,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 1503,
     "records": 1503,
     "too_few": false
    }
   },
   "Qwen3.8 Flash (off)": {
    "chosen": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.37252926284163007,
       "n": 23,
       "observed": 0.30434782608695654
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.4556349964531084,
       "n": 98,
       "observed": 0.46938775510204084
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.548649542416161,
       "n": 165,
       "observed": 0.5636363636363636
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.647204987294955,
       "n": 138,
       "observed": 0.6521739130434783
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7466425533224349,
       "n": 130,
       "observed": 0.7307692307692307
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8335058373337157,
       "n": 61,
       "observed": 0.819672131147541
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": null,
       "n": 0,
       "observed": null
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.29434337406010297,
       "n": 3,
       "observed": 0.6666666666666666
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.3688878689838833,
       "n": 31,
       "observed": 0.2903225806451613
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.45466986227992945,
       "n": 95,
       "observed": 0.5052631578947369
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5505625736354633,
       "n": 146,
       "observed": 0.5547945205479452
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6464114884643934,
       "n": 118,
       "observed": 0.6271186440677966
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7516474683648178,
       "n": 134,
       "observed": 0.6940298507462687
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8423549783904305,
       "n": 84,
       "observed": 0.8333333333333334
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9141638349229477,
       "n": 4,
       "observed": 1.0
      }
     ],
     "brier_after": 0.22058907923728266,
     "brier_before": 0.21990012444258,
     "event_rate": 0.6195121951219512,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 615,
     "records": 615,
     "too_few": false
    },
    "risk_class": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.16768016096492844,
       "n": 11,
       "observed": 0.36363636363636365
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.2607814170231341,
       "n": 135,
       "observed": 0.25925925925925924
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.34226855987979427,
       "n": 100,
       "observed": 0.33
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.4248318734377681,
       "n": 21,
       "observed": 0.3333333333333333
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5411495359461062,
       "n": 5,
       "observed": 0.8
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": null,
       "n": 0,
       "observed": null
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.06237826675145734,
       "n": 25,
       "observed": 0.28
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.15475108263857,
       "n": 61,
       "observed": 0.22950819672131148
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.25012202638841174,
       "n": 62,
       "observed": 0.27419354838709675
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": 0.3489524726999153,
       "n": 28,
       "observed": 0.2857142857142857
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": 0.4502872614244484,
       "n": 26,
       "observed": 0.46153846153846156
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5453432663797596,
       "n": 22,
       "observed": 0.2727272727272727
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6394790114552671,
       "n": 20,
       "observed": 0.35
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7459187211618604,
       "n": 17,
       "observed": 0.29411764705882354
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8333989894060008,
       "n": 7,
       "observed": 0.42857142857142855
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9403696242344715,
       "n": 4,
       "observed": 1.0
      }
     ],
     "brier_after": 0.20969267690643048,
     "brier_before": 0.23681517868733964,
     "event_rate": 0.30514705882352944,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 272,
     "records": 272,
     "too_few": false
    }
   },
   "Qwen3.8-27B (low)": {
    "chosen": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.6817612665532018,
       "n": 87,
       "observed": 0.735632183908046
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7313896914882994,
       "n": 210,
       "observed": 0.719047619047619
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": 0.8031907901647155,
       "n": 1,
       "observed": 0.0
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": null,
       "n": 0,
       "observed": null
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9998077571462516,
       "n": 298,
       "observed": 0.7214765100671141
      }
     ],
     "brier_after": 0.20345092715978055,
     "brier_before": 0.27843161510564046,
     "event_rate": 0.7214765100671141,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 298,
     "records": 298,
     "too_few": false
    },
    "risk_class": {
     "bins_after": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": 0.17977928083680464,
       "n": 56,
       "observed": 0.23214285714285715
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": 0.2134397367025193,
       "n": 46,
       "observed": 0.15217391304347827
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": 0.5739717050099056,
       "n": 8,
       "observed": 0.75
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": 0.640952698225523,
       "n": 7,
       "observed": 0.7142857142857143
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": 0.7208609262559564,
       "n": 6,
       "observed": 0.3333333333333333
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": null,
       "n": 0,
       "observed": null
      }
     ],
     "bins_before": [
      {
       "hi": 0.1,
       "lo": 0.0,
       "mean_p": 0.00010206086801754403,
       "n": 102,
       "observed": 0.19607843137254902
      },
      {
       "hi": 0.2,
       "lo": 0.1,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.3,
       "lo": 0.2,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.4,
       "lo": 0.3,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.5,
       "lo": 0.4,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.6,
       "lo": 0.5,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.7,
       "lo": 0.6,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.8,
       "lo": 0.7,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 0.9,
       "lo": 0.8,
       "mean_p": null,
       "n": 0,
       "observed": null
      },
      {
       "hi": 1.0,
       "lo": 0.9,
       "mean_p": 0.9999173277525093,
       "n": 21,
       "observed": 0.6190476190476191
      }
     ],
     "brier_after": 0.1769989393543457,
     "brier_before": 0.2275834104749831,
     "event_rate": 0.2682926829268293,
     "failed_folds": 0,
     "left_out": 0,
     "paired": 123,
     "records": 123,
     "too_few": true
    }
   }
  },
  "cheapest": {
   "bar_error_rate": 35,
   "bar_judgement": 60,
   "chosen_splits": 300,
   "cleared_splits": 298,
   "coverage": {
    "Claude Fable 5.1 (low)": {
     "decisions": 2008,
     "priced": 2008
    },
    "Claude Sonnet 5.5 (high)": {
     "decisions": 2164,
     "priced": 2164
    },
    "Claude Sonnet 5.5 (low)": {
     "decisions": 2150,
     "priced": 2150
    },
    "Clef": {
     "decisions": 1978,
     "priced": 0
    },
    "DeepSeek-V4.1-Flash (low)": {
     "decisions": 2226,
     "priced": 2226
    },
    "DeepSeek-V4.1-Flash 1017 (low)": {
     "decisions": 2234,
     "priced": 2234
    },
    "DeepSeek-V4.1-Flash 1017 (off)": {
     "decisions": 2034,
     "priced": 2034
    },
    "Fastino GLiDE": {
     "decisions": 2159,
     "priced": 0
    },
    "GLM-5.3 (low)": {
     "decisions": 1989,
     "priced": 1989
    },
    "Gemini 3.8 Flash (high)": {
     "decisions": 2168,
     "priced": 2168
    },
    "Gemini 3.8 Flash (low)": {
     "decisions": 2053,
     "priced": 2053
    },
    "Gemma 4 31B (off)": {
     "decisions": 1847,
     "priced": 0
    },
    "Gemma 4 31B (on)": {
     "decisions": 2122,
     "priced": 0
    },
    "Grok 4.7 (low)": {
     "decisions": 2152,
     "priced": 2152
    },
    "Jev 1.14.0": {
     "decisions": 1912,
     "priced": 1912
    },
    "Liquid d1-ultra": {
     "decisions": 2071,
     "priced": 2071
    },
    "Qwen3.8 Flash (off)": {
     "decisions": 2027,
     "priced": 2027
    },
    "Qwen3.8-27B (low)": {
     "decisions": 2108,
     "priced": 1
    }
   },
   "no_priced_qualifier": 0,
   "no_qualifier": 0,
   "setups": {
    "Claude Fable 5.1 (low)": {
     "chosen": 0,
     "cleared": 0,
     "qualified": 300,
     "qualified_unpriced": 0,
     "test_na": 0
    },
    "Claude Sonnet 5.5 (high)": {
     "chosen": 0,
     "cleared": 0,
     "qualified": 300,
     "qualified_unpriced": 0,
     "test_na": 0
    },
    "Claude Sonnet 5.5 (low)": {
     "chosen": 0,
     "cleared": 0,
     "qualified": 300,
     "qualified_unpriced": 0,
     "test_na": 0
    },
    "Clef": {
     "chosen": 0,
     "cleared": 0,
     "qualified": 2,
     "qualified_unpriced": 2,
     "test_na": 0
    },
    "DeepSeek-V4.1-Flash (low)": {
     "chosen": 0,
     "cleared": 0,
     "qualified": 300,
     "qualified_unpriced": 0,
     "test_na": 0
    },
    "DeepSeek-V4.1-Flash 1017 (low)": {
     "chosen": 0,
     "cleared": 0,
     "qualified": 300,
     "qualified_unpriced": 0,
     "test_na": 0
    },
    "DeepSeek-V4.1-Flash 1017 (off)": {
     "chosen": 0,
     "cleared": 0,
     "qualified": 21,
     "qualified_unpriced": 0,
     "test_na": 0
    },
    "Fastino GLiDE": {
     "chosen": 0,
     "cleared": 0,
     "qualified": 190,
     "qualified_unpriced": 190,
     "test_na": 0
    },
    "GLM-5.3 (low)": {
     "chosen": 0,
     "cleared": 0,
     "qualified": 292,
     "qualified_unpriced": 0,
     "test_na": 0
    },
    "Gemini 3.8 Flash (high)": {
     "chosen": 0,
     "cleared": 0,
     "qualified": 300,
     "qualified_unpriced": 0,
     "test_na": 0
    },
    "Gemini 3.8 Flash (low)": {
     "chosen": 0,
     "cleared": 0,
     "qualified": 300,
     "qualified_unpriced": 0,
     "test_na": 0
    },
    "Gemma 4 31B (off)": {
     "chosen": 0,
     "cleared": 0,
     "qualified": 295,
     "qualified_unpriced": 295,
     "test_na": 0
    },
    "Gemma 4 31B (on)": {
     "chosen": 0,
     "cleared": 0,
     "qualified": 300,
     "qualified_unpriced": 300,
     "test_na": 0
    },
    "Grok 4.7 (low)": {
     "chosen": 0,
     "cleared": 0,
     "qualified": 300,
     "qualified_unpriced": 0,
     "test_na": 0
    },
    "Jev 1.14.0": {
     "chosen": 1,
     "cleared": 0,
     "qualified": 81,
     "qualified_unpriced": 0,
     "test_na": 0
    },
    "Liquid d1-ultra": {
     "chosen": 299,
     "cleared": 298,
     "qualified": 299,
     "qualified_unpriced": 0,
     "test_na": 0
    },
    "Qwen3.8 Flash (off)": {
     "chosen": 0,
     "cleared": 0,
     "qualified": 93,
     "qualified_unpriced": 0,
     "test_na": 0
    },
    "Qwen3.8-27B (low)": {
     "chosen": 0,
     "cleared": 0,
     "qualified": 287,
     "qualified_unpriced": 287,
     "test_na": 0
    }
   },
   "splits": 300
  },
  "distributions": {
   "Clef": {
    "bad_risk": {
     "counts": [
      8,
      39,
      22,
      15,
      4,
      0,
      1,
      0,
      0,
      0
     ],
     "mean": 0.21371348314606745,
     "median": 0.1854,
     "n": 89,
     "too_few": true
    },
    "right": {
     "counts": [
      6,
      57,
      45,
      35,
      42,
      23,
      15,
      8,
      0,
      0
     ],
     "mean": 0.34247142857142865,
     "median": 0.3238,
     "n": 231,
     "too_few": false
    },
    "right_risk": {
     "counts": [
      5,
      38,
      28,
      5,
      2,
      1,
      1,
      0,
      0,
      0
     ],
     "mean": 0.20312125,
     "median": 0.1964,
     "n": 80,
     "too_few": true
    },
    "wrong": {
     "counts": [
      26,
      119,
      75,
      40,
      9,
      5,
      4,
      0,
      0,
      0
     ],
     "mean": 0.2182615107913669,
     "median": 0.1926,
     "n": 278,
     "too_few": false
    }
   },
   "DeepSeek-V4.1-Flash (low)": {
    "bad_risk": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      11
     ],
     "mean": 0.9998802196300638,
     "median": 0.9999485577615669,
     "n": 11,
     "too_few": true
    },
    "right": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      325
     ],
     "mean": 0.9999105129200195,
     "median": 0.9999940130571493,
     "n": 325,
     "too_few": false
    },
    "right_risk": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      19
     ],
     "mean": 0.9998634542955809,
     "median": 0.9999479164287769,
     "n": 19,
     "too_few": true
    },
    "wrong": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      83
     ],
     "mean": 0.9999172698372321,
     "median": 0.9999938558224682,
     "n": 83,
     "too_few": true
    }
   },
   "DeepSeek-V4.1-Flash 1017 (low)": {
    "bad_risk": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      2
     ],
     "mean": 0.9999999621078929,
     "median": 0.9999999621078929,
     "n": 2,
     "too_few": true
    },
    "right": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      150
     ],
     "mean": 0.9999988713475182,
     "median": 0.9999995598218907,
     "n": 150,
     "too_few": true
    },
    "right_risk": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      7
     ],
     "mean": 0.9999986861962805,
     "median": 0.9999990714080635,
     "n": 7,
     "too_few": true
    },
    "wrong": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      30
     ],
     "mean": 0.999998786005652,
     "median": 0.9999994703439594,
     "n": 30,
     "too_few": true
    }
   },
   "DeepSeek-V4.1-Flash 1017 (off)": {
    "bad_risk": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      1,
      1,
      3,
      3,
      6
     ],
     "mean": 0.8482643870875163,
     "median": 0.8684971667996171,
     "n": 14,
     "too_few": true
    },
    "right": {
     "counts": [
      0,
      0,
      0,
      3,
      2,
      11,
      16,
      21,
      30,
      278
     ],
     "mean": 0.9260789455950884,
     "median": 0.9947491095551243,
     "n": 361,
     "too_few": false
    },
    "right_risk": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      3,
      1,
      3,
      1,
      4
     ],
     "mean": 0.7862564526112089,
     "median": 0.7895431350931046,
     "n": 12,
     "too_few": true
    },
    "wrong": {
     "counts": [
      0,
      0,
      0,
      2,
      4,
      26,
      13,
      22,
      17,
      92
     ],
     "mean": 0.8352485000709106,
     "median": 0.9340250910593297,
     "n": 176,
     "too_few": true
    }
   },
   "Fastino GLiDE": {
    "bad_risk": {
     "counts": [
      8,
      36,
      61,
      41,
      26,
      16,
      11,
      3,
      0,
      0
     ],
     "mean": 0.32720859275219677,
     "median": 0.29276430811586207,
     "n": 202,
     "too_few": false
    },
    "right": {
     "counts": [
      1,
      51,
      105,
      134,
      142,
      153,
      133,
      106,
      94,
      26
     ],
     "mean": 0.5280649992234184,
     "median": 0.5266723360920266,
     "n": 945,
     "too_few": false
    },
    "right_risk": {
     "counts": [
      2,
      26,
      61,
      43,
      43,
      20,
      21,
      6,
      3,
      0
     ],
     "mean": 0.37588558214606393,
     "median": 0.3475663938850449,
     "n": 225,
     "too_few": false
    },
    "wrong": {
     "counts": [
      30,
      142,
      252,
      187,
      155,
      124,
      58,
      40,
      11,
      3
     ],
     "mean": 0.3691658141153888,
     "median": 0.3462324634121505,
     "n": 1002,
     "too_few": false
    }
   },
   "GLM-5.3 (low)": {
    "bad_risk": {
     "counts": [
      0,
      0,
      0,
      0,
      1,
      1,
      3,
      2,
      2,
      9
     ],
     "mean": 0.8376175327691532,
     "median": 0.9199528074681611,
     "n": 18,
     "too_few": true
    },
    "right": {
     "counts": [
      0,
      0,
      0,
      0,
      3,
      7,
      4,
      17,
      26,
      299
     ],
     "mean": 0.9435685334602296,
     "median": 0.9838038440014706,
     "n": 356,
     "too_few": false
    },
    "right_risk": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      4,
      0,
      3,
      0,
      10
     ],
     "mean": 0.838142623909786,
     "median": 0.9740426405266815,
     "n": 17,
     "too_few": true
    },
    "wrong": {
     "counts": [
      0,
      0,
      0,
      2,
      6,
      8,
      8,
      7,
      9,
      131
     ],
     "mean": 0.9040110762678557,
     "median": 0.9767546146098207,
     "n": 171,
     "too_few": true
    }
   },
   "Gemma 4 31B (off)": {
    "bad_risk": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      3,
      1,
      0,
      1,
      14
     ],
     "mean": 0.8911127484569906,
     "median": 0.9959187248202286,
     "n": 19,
     "too_few": true
    },
    "right": {
     "counts": [
      0,
      0,
      0,
      0,
      1,
      5,
      6,
      6,
      8,
      290
     ],
     "mean": 0.9717830205992132,
     "median": 0.9999519781965955,
     "n": 316,
     "too_few": false
    },
    "right_risk": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      2,
      0,
      0,
      2,
      23
     ],
     "mean": 0.9424945610548192,
     "median": 0.9922858147447025,
     "n": 27,
     "too_few": true
    },
    "wrong": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      4,
      4,
      4,
      12,
      135
     ],
     "mean": 0.9504194276681004,
     "median": 0.9978170418112289,
     "n": 159,
     "too_few": true
    }
   },
   "Gemma 4 31B (on)": {
    "bad_risk": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0
     ],
     "mean": null,
     "median": null,
     "n": 0,
     "too_few": true
    },
    "right": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      142
     ],
     "mean": 0.9999857471496028,
     "median": 0.9999999226567585,
     "n": 142,
     "too_few": true
    },
    "right_risk": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      3
     ],
     "mean": 0.99943271648816,
     "median": 0.999999694105726,
     "n": 3,
     "too_few": true
    },
    "wrong": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      27
     ],
     "mean": 0.9999086064038655,
     "median": 0.9999996940991632,
     "n": 27,
     "too_few": true
    }
   },
   "Jev 1.14.0": {
    "bad_risk": {
     "counts": [
      0,
      1,
      9,
      13,
      9,
      4,
      2,
      0,
      0,
      0
     ],
     "mean": 0.3794736842105264,
     "median": 0.37,
     "n": 38,
     "too_few": true
    },
    "right": {
     "counts": [
      0,
      2,
      13,
      29,
      33,
      34,
      34,
      47,
      44,
      9
     ],
     "mean": 0.6112244897959184,
     "median": 0.63,
     "n": 245,
     "too_few": false
    },
    "right_risk": {
     "counts": [
      0,
      4,
      12,
      15,
      11,
      8,
      6,
      1,
      1,
      0
     ],
     "mean": 0.4055172413793104,
     "median": 0.37,
     "n": 58,
     "too_few": true
    },
    "wrong": {
     "counts": [
      0,
      8,
      41,
      57,
      63,
      39,
      26,
      21,
      5,
      2
     ],
     "mean": 0.4517557251908397,
     "median": 0.43,
     "n": 262,
     "too_few": false
    }
   },
   "Liquid d1-ultra": {
    "bad_risk": {
     "counts": [
      1,
      48,
      56,
      35,
      21,
      7,
      2,
      1,
      1,
      0
     ],
     "mean": 0.2885489114565216,
     "median": 0.2505627513116955,
     "n": 172,
     "too_few": true
    },
    "right": {
     "counts": [
      2,
      51,
      176,
      175,
      158,
      152,
      125,
      90,
      58,
      19
     ],
     "mean": 0.4820643544205223,
     "median": 0.466486917754962,
     "n": 1006,
     "too_few": false
    },
    "right_risk": {
     "counts": [
      1,
      40,
      96,
      53,
      38,
      17,
      15,
      20,
      8,
      1
     ],
     "mean": 0.3735032791411418,
     "median": 0.31683448979435236,
     "n": 289,
     "too_few": false
    },
    "wrong": {
     "counts": [
      13,
      226,
      359,
      206,
      129,
      65,
      36,
      19,
      10,
      1
     ],
     "mean": 0.3171151857850045,
     "median": 0.27628877600361046,
     "n": 1064,
     "too_few": false
    }
   },
   "Qwen3.8 Flash (off)": {
    "bad_risk": {
     "counts": [
      0,
      0,
      1,
      7,
      5,
      14,
      4,
      5,
      0,
      0
     ],
     "mean": 0.5289990751178595,
     "median": 0.53065248984668,
     "n": 36,
     "too_few": true
    },
    "right": {
     "counts": [
      0,
      0,
      2,
      9,
      48,
      81,
      74,
      93,
      70,
      4
     ],
     "mean": 0.6595499952290556,
     "median": 0.6624117872314064,
     "n": 381,
     "too_few": false
    },
    "right_risk": {
     "counts": [
      0,
      0,
      0,
      2,
      2,
      3,
      2,
      4,
      1,
      0
     ],
     "mean": 0.5903356105574932,
     "median": 0.5748397643434613,
     "n": 14,
     "too_few": true
    },
    "wrong": {
     "counts": [
      0,
      0,
      1,
      22,
      47,
      65,
      44,
      41,
      14,
      0
     ],
     "mean": 0.5812716601280062,
     "median": 0.566836140939781,
     "n": 234,
     "too_few": false
    }
   },
   "Qwen3.8-27B (low)": {
    "bad_risk": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      9
     ],
     "mean": 0.9998516457353543,
     "median": 0.9998496881470403,
     "n": 9,
     "too_few": true
    },
    "right": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      215
     ],
     "mean": 0.9997974557541832,
     "median": 0.9998941890108743,
     "n": 215,
     "too_few": false
    },
    "right_risk": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      12
     ],
     "mean": 0.9998764425100225,
     "median": 0.9998911825387569,
     "n": 12,
     "too_few": true
    },
    "wrong": {
     "counts": [
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      0,
      83
     ],
     "mean": 0.999834441475103,
     "median": 0.9998878222347979,
     "n": 83,
     "too_few": true
    }
   }
  },
  "hypothesis": {
   "X": {
    "Clef": -0.08553165915210409,
    "DeepSeek-V4.1-Flash (low)": 0.5020711547664719,
    "DeepSeek-V4.1-Flash 1017 (off)": 0.9119532851968042,
    "Fastino GLiDE": -0.24909767282537112,
    "GLM-5.3 (low)": 1.080474975041219,
    "Gemma 4 31B (off)": 0.6317237592080054,
    "Jev 1.14.0": 0.673495507487854,
    "Liquid d1-ultra": 0.606517325681844,
    "Qwen3.8 Flash (off)": 0.41199492132134463,
    "Qwen3.8-27B (low)": 0.678333261532987
   },
   "Y": {
    "Clef": 1.0984978241707526,
    "DeepSeek-V4.1-Flash (low)": 0.0,
    "DeepSeek-V4.1-Flash 1017 (off)": 6.145350125643846,
    "Fastino GLiDE": -0.30101246439206714,
    "GLM-5.3 (low)": -2.300979002401256,
    "Gemma 4 31B (off)": 0.38863305845804047,
    "Jev 1.14.0": 2.770577772468593,
    "Liquid d1-ultra": -1.5315700222809912,
    "Qwen3.8 Flash (off)": 0.5307841281143474,
    "Qwen3.8-27B (low)": 0.0
   }
  },
  "label": "re-read from the model's probabilities, not played",
  "reread": {
   "Clef": {
    "always_risky": 0,
    "always_safe": 0,
    "chosen": {
     "0.75": 66,
     "0.80": 173,
     "0.85": 60,
     "0.90": 1
    },
    "judgement_diff": 1.0984978241707526,
    "judgement_diff_hi": 4.745219147512182,
    "judgement_diff_lo": -3.3283937349153767,
    "judgement_diff_p5": -3.02757175780432,
    "judgement_diff_p95": 5.4384588812655,
    "judgement_played": 58.95124075744457,
    "judgement_reread": 60.04973858161534,
    "no_fit": 0,
    "pr_diff": 5.922551443753456,
    "pr_diff_hi": 8.28936760947194,
    "pr_diff_lo": 2.6808737429840144,
    "pr_diff_p5": 2.4169153624942306,
    "pr_diff_p95": 9.979995123248823,
    "pr_played": 37.78761852050686,
    "pr_reread": 43.71016996426031,
    "predictor": -0.08553165915210409,
    "predictor_splits": 300,
    "records": 509,
    "replicates": 200,
    "test_judgement_na": 0,
    "test_records_mean": 250.51,
    "trade_off": 382
   },
   "DeepSeek-V4.1-Flash (low)": {
    "always_risky": 0,
    "always_safe": 0,
    "chosen": {
     "0.50": 300
    },
    "judgement_diff": 0.0,
    "judgement_diff_hi": 0.0,
    "judgement_diff_lo": -0.7515598893065991,
    "judgement_diff_p5": 0.0,
    "judgement_diff_p95": 0.0,
    "judgement_played": 64.06487998947787,
    "judgement_reread": 64.06487998947787,
    "no_fit": 0,
    "pr_diff": 0.0,
    "pr_diff_hi": 0.3536029636060406,
    "pr_diff_lo": 0.0,
    "pr_diff_p5": 0.0,
    "pr_diff_p95": 0.0,
    "pr_played": 7.23172275215528,
    "pr_reread": 7.23172275215528,
    "predictor": 0.5020711547664719,
    "predictor_splits": 300,
    "records": 408,
    "replicates": 200,
    "test_judgement_na": 0,
    "test_records_mean": 204.82666666666665,
    "trade_off": 204
   },
   "DeepSeek-V4.1-Flash 1017 (off)": {
    "always_risky": 0,
    "always_safe": 0,
    "chosen": {
     "0.05": 117,
     "0.10": 69,
     "0.15": 45,
     "0.20": 61,
     "0.25": 5,
     "0.40": 2,
     "0.65": 1
    },
    "judgement_diff": 6.145350125643846,
    "judgement_diff_hi": 11.496313716167078,
    "judgement_diff_lo": 0.4430504120719478,
    "judgement_diff_p5": 1.631181719414595,
    "judgement_diff_p95": 10.42194609291947,
    "judgement_played": 54.71008054356092,
    "judgement_reread": 60.85543066920477,
    "no_fit": 0,
    "pr_diff": 0.010450518870872757,
    "pr_diff_hi": 2.420093199672782,
    "pr_diff_lo": -3.2101932716970563,
    "pr_diff_p5": -3.0302425075484347,
    "pr_diff_p95": 3.9378332620008516,
    "pr_played": 15.107917156644907,
    "pr_reread": 15.118367675515783,
    "predictor": 0.9119532851968042,
    "predictor_splits": 300,
    "records": 537,
    "replicates": 200,
    "test_judgement_na": 0,
    "test_records_mean": 263.9,
    "trade_off": 246
   },
   "Fastino GLiDE": {
    "always_risky": 0,
    "always_safe": 0,
    "chosen": {
     "0.55": 171,
     "0.60": 52,
     "0.65": 6,
     "0.70": 17,
     "0.75": 53,
     "0.80": 1
    },
    "judgement_diff": -0.30101246439206714,
    "judgement_diff_hi": 1.639227455787445,
    "judgement_diff_lo": -2.19477009093377,
    "judgement_diff_p5": -1.9046765541841453,
    "judgement_diff_p95": 1.3051742911940314,
    "judgement_played": 60.3428490949337,
    "judgement_reread": 60.04183663054163,
    "no_fit": 0,
    "pr_diff": 4.428187978812992,
    "pr_diff_hi": 6.040083136011141,
    "pr_diff_lo": 1.5901908502830973,
    "pr_diff_p5": 1.269846102845171,
    "pr_diff_p95": 7.449863282859269,
    "pr_played": 33.41316412976402,
    "pr_reread": 37.84135210857701,
    "predictor": -0.24909767282537112,
    "predictor_splits": 300,
    "records": 1947,
    "replicates": 200,
    "test_judgement_na": 0,
    "test_records_mean": 963.68,
    "trade_off": 1426
   },
   "GLM-5.3 (low)": {
    "always_risky": 0,
    "always_safe": 0,
    "chosen": {
     "0.05": 15,
     "0.10": 55,
     "0.15": 12,
     "0.25": 23,
     "0.50": 99,
     "0.65": 1,
     "0.70": 70,
     "0.75": 18,
     "0.95": 6,
     "1.00": 1
    },
    "judgement_diff": -2.300979002401256,
    "judgement_diff_hi": -0.06513434598459106,
    "judgement_diff_lo": -4.583659325048334,
    "judgement_diff_p5": -6.10173319879482,
    "judgement_diff_p95": 0.0,
    "judgement_played": 56.84591969860424,
    "judgement_reread": 54.54494069620298,
    "no_fit": 0,
    "pr_diff": 1.9544655149753392,
    "pr_diff_hi": 5.23511138350367,
    "pr_diff_lo": 0.3422208691776814,
    "pr_diff_p5": -0.2152946743294235,
    "pr_diff_p95": 7.631664919980162,
    "pr_played": 14.22688694812192,
    "pr_reread": 16.18135246309726,
    "predictor": 1.080474975041219,
    "predictor_splits": 300,
    "records": 527,
    "replicates": 200,
    "test_judgement_na": 0,
    "test_records_mean": 261.55,
    "trade_off": 253
   },
   "Gemma 4 31B (off)": {
    "always_risky": 0,
    "always_safe": 0,
    "chosen": {
     "0.15": 200,
     "0.20": 11,
     "0.50": 14,
     "0.60": 3,
     "0.65": 14,
     "0.85": 34,
     "0.95": 24
    },
    "judgement_diff": 0.38863305845804047,
    "judgement_diff_hi": 2.0734406771904323,
    "judgement_diff_lo": -1.31318144558948,
    "judgement_diff_p5": -1.6867584047462147,
    "judgement_diff_p95": 2.139800514800522,
    "judgement_played": 61.885049871178076,
    "judgement_reread": 62.273682929636124,
    "no_fit": 0,
    "pr_diff": 0.2151717532316345,
    "pr_diff_hi": 0.7318697728359703,
    "pr_diff_lo": -0.36715868674809055,
    "pr_diff_p5": -0.44459724729420413,
    "pr_diff_p95": 0.8302185572913269,
    "pr_played": 11.971104257202574,
    "pr_reread": 12.18627601043421,
    "predictor": 0.6317237592080054,
    "predictor_splits": 300,
    "records": 475,
    "replicates": 200,
    "test_judgement_na": 0,
    "test_records_mean": 240.14666666666668,
    "trade_off": 225
   },
   "Jev 1.14.0": {
    "always_risky": 0,
    "always_safe": 0,
    "chosen": {
     "0.40": 9,
     "0.45": 88,
     "0.50": 187,
     "0.55": 1,
     "0.60": 8,
     "0.65": 2,
     "0.85": 5
    },
    "judgement_diff": 2.770577772468593,
    "judgement_diff_hi": 5.975535626023067,
    "judgement_diff_lo": -1.5071411144756175,
    "judgement_diff_p5": -0.15139453477869208,
    "judgement_diff_p95": 5.780274755977497,
    "judgement_played": 60.2092852162709,
    "judgement_reread": 62.97986298873948,
    "no_fit": 0,
    "pr_diff": 0.7472320974479474,
    "pr_diff_hi": 3.384214284437627,
    "pr_diff_lo": -1.1797041758312332,
    "pr_diff_p5": -1.4982950563232116,
    "pr_diff_p95": 3.0223549891641177,
    "pr_played": 29.84932486785434,
    "pr_reread": 30.59655696530228,
    "predictor": 0.673495507487854,
    "predictor_splits": 300,
    "records": 507,
    "replicates": 200,
    "test_judgement_na": 0,
    "test_records_mean": 248.34333333333333,
    "trade_off": 367
   },
   "Liquid d1-ultra": {
    "always_risky": 0,
    "always_safe": 0,
    "chosen": {
     "0.55": 1,
     "0.60": 15,
     "0.65": 136,
     "0.70": 60,
     "0.75": 86,
     "0.80": 2
    },
    "judgement_diff": -1.5315700222809912,
    "judgement_diff_hi": 0.38654177112938815,
    "judgement_diff_lo": -3.286342596665198,
    "judgement_diff_p5": -2.8983356370522833,
    "judgement_diff_p95": -0.24929702408887933,
    "judgement_played": 64.09398654271044,
    "judgement_reread": 62.562416520429444,
    "no_fit": 0,
    "pr_diff": 3.494711338126811,
    "pr_diff_hi": 5.415087624327504,
    "pr_diff_lo": 1.7050173672890179,
    "pr_diff_p5": 1.8843441831549748,
    "pr_diff_p95": 5.143193938515793,
    "pr_played": 30.3270721764501,
    "pr_reread": 33.82178351457691,
    "predictor": 0.606517325681844,
    "predictor_splits": 300,
    "records": 2070,
    "replicates": 200,
    "test_judgement_na": 0,
    "test_records_mean": 1041.4966666666667,
    "trade_off": 1503
   },
   "Qwen3.8 Flash (off)": {
    "always_risky": 0,
    "always_safe": 0,
    "chosen": {
     "0.20": 1,
     "0.30": 7,
     "0.35": 77,
     "0.40": 21,
     "0.45": 16,
     "0.50": 1,
     "0.60": 27,
     "0.65": 1,
     "0.70": 24,
     "0.80": 43,
     "0.85": 52,
     "0.90": 30
    },
    "judgement_diff": 0.5307841281143474,
    "judgement_diff_hi": 4.292500362076111,
    "judgement_diff_lo": -2.6926645480194247,
    "judgement_diff_p5": -2.8942225863611637,
    "judgement_diff_p95": 3.7499136400126445,
    "judgement_played": 49.426893410071656,
    "judgement_reread": 49.957677538186,
    "no_fit": 0,
    "pr_diff": 1.7818047028876902,
    "pr_diff_hi": 8.744782881376635,
    "pr_diff_lo": -1.7824776332206513,
    "pr_diff_p5": -4.116488234383911,
    "pr_diff_p95": 11.217825840386883,
    "pr_played": 16.6976510282046,
    "pr_reread": 18.47945573109229,
    "predictor": 0.41199492132134463,
    "predictor_splits": 300,
    "records": 615,
    "replicates": 200,
    "test_judgement_na": 0,
    "test_records_mean": 309.62,
    "trade_off": 272
   },
   "Qwen3.8-27B (low)": {
    "always_risky": 0,
    "always_safe": 0,
    "chosen": {
     "0.50": 300
    },
    "judgement_diff": 0.0,
    "judgement_diff_hi": 0.0,
    "judgement_diff_lo": -5.767709860363101,
    "judgement_diff_p5": 0.0,
    "judgement_diff_p95": 0.0,
    "judgement_played": 63.21519214542774,
    "judgement_reread": 63.21519214542774,
    "no_fit": 0,
    "pr_diff": 0.0,
    "pr_diff_hi": 1.4177093999507469,
    "pr_diff_lo": -0.18222489427529937,
    "pr_diff_p5": 0.0,
    "pr_diff_p95": 0.0,
    "pr_played": 15.613139960197016,
    "pr_reread": 15.613139960197016,
    "predictor": 0.678333261532987,
    "predictor_splits": 300,
    "records": 298,
    "replicates": 200,
    "test_judgement_na": 0,
    "test_records_mean": 147.91666666666666,
    "trade_off": 123
   }
  },
  "reread_too_few": {
   "DeepSeek-V4.1-Flash 1017 (low)": 180,
   "Gemma 4 31B (on)": 169
  },
  "spec_fixed_on": "2026-10-24T12:55Z",
  "splits": 300,
  "support": {
   "Claude Fable 5.1 (low)": {
    "complete": 0,
    "grouped": 0,
    "invalid": 0,
    "no_probabilities": 1811,
    "no_raw": 0,
    "not_audited": 197,
    "sparse": 0,
    "unverified": 0
   },
   "Claude Sonnet 5.5 (high)": {
    "complete": 0,
    "grouped": 0,
    "invalid": 0,
    "no_probabilities": 1991,
    "no_raw": 0,
    "not_audited": 173,
    "sparse": 0,
    "unverified": 0
   },
   "Claude Sonnet 5.5 (low)": {
    "complete": 0,
    "grouped": 0,
    "invalid": 0,
    "no_probabilities": 1941,
    "no_raw": 0,
    "not_audited": 209,
    "sparse": 0,
    "unverified": 0
   },
   "Clef": {
    "complete": 509,
    "grouped": 3,
    "invalid": 0,
    "no_probabilities": 0,
    "no_raw": 0,
    "not_audited": 1466,
    "sparse": 0,
    "unverified": 0
   },
   "DeepSeek-V4.1-Flash (low)": {
    "complete": 408,
    "grouped": 245,
    "invalid": 38,
    "no_probabilities": 0,
    "no_raw": 0,
    "not_audited": 265,
    "sparse": 1270,
    "unverified": 0
   },
   "DeepSeek-V4.1-Flash 1017 (low)": {
    "complete": 180,
    "grouped": 204,
    "invalid": 12,
    "no_probabilities": 0,
    "no_raw": 0,
    "not_audited": 527,
    "sparse": 1311,
    "unverified": 0
   },
   "DeepSeek-V4.1-Flash 1017 (off)": {
    "complete": 537,
    "grouped": 128,
    "invalid": 0,
    "no_probabilities": 0,
    "no_raw": 0,
    "not_audited": 428,
    "sparse": 941,
    "unverified": 0
   },
   "Fastino GLiDE": {
    "complete": 1947,
    "grouped": 2,
    "invalid": 0,
    "no_probabilities": 0,
    "no_raw": 0,
    "not_audited": 210,
    "sparse": 0,
    "unverified": 0
   },
   "GLM-5.3 (low)": {
    "complete": 527,
    "grouped": 184,
    "invalid": 0,
    "no_probabilities": 1,
    "no_raw": 0,
    "not_audited": 187,
    "sparse": 1090,
    "unverified": 0
   },
   "GNUbg weak": {
    "complete": 0,
    "grouped": 0,
    "invalid": 0,
    "no_probabilities": 2136,
    "no_raw": 0,
    "not_audited": 0,
    "sparse": 0,
    "unverified": 0
   },
   "Gemini 3.8 Flash (high)": {
    "complete": 0,
    "grouped": 0,
    "invalid": 0,
    "no_probabilities": 1974,
    "no_raw": 0,
    "not_audited": 194,
    "sparse": 0,
    "unverified": 0
   },
   "Gemini 3.8 Flash (low)": {
    "complete": 0,
    "grouped": 0,
    "invalid": 0,
    "no_probabilities": 1830,
    "no_raw": 0,
    "not_audited": 223,
    "sparse": 0,
    "unverified": 0
   },
   "Gemma 4 31B (off)": {
    "complete": 475,
    "grouped": 148,
    "invalid": 0,
    "no_probabilities": 0,
    "no_raw": 0,
    "not_audited": 231,
    "sparse": 993,
    "unverified": 0
   },
   "Gemma 4 31B (on)": {
    "complete": 169,
    "grouped": 175,
    "invalid": 0,
    "no_probabilities": 0,
    "no_raw": 0,
    "not_audited": 599,
    "sparse": 1179,
    "unverified": 0
   },
   "Grok 4.7 (low)": {
    "complete": 0,
    "grouped": 0,
    "invalid": 0,
    "no_probabilities": 1741,
    "no_raw": 0,
    "not_audited": 411,
    "sparse": 0,
    "unverified": 0
   },
   "Jev 1.14.0": {
    "complete": 507,
    "grouped": 0,
    "invalid": 0,
    "no_probabilities": 0,
    "no_raw": 0,
    "not_audited": 1405,
    "sparse": 0,
    "unverified": 0
   },
   "Liquid d1-ultra": {
    "complete": 2070,
    "grouped": 1,
    "invalid": 0,
    "no_probabilities": 0,
    "no_raw": 0,
    "not_audited": 0,
    "sparse": 0,
    "unverified": 0
   },
   "Qwen3.8 Flash (off)": {
    "complete": 615,
    "grouped": 145,
    "invalid": 0,
    "no_probabilities": 0,
    "no_raw": 0,
    "not_audited": 243,
    "sparse": 1024,
    "unverified": 0
   },
   "Qwen3.8-27B (low)": {
    "complete": 298,
    "grouped": 142,
    "invalid": 0,
    "no_probabilities": 0,
    "no_raw": 0,
    "not_audited": 729,
    "sparse": 939,
    "unverified": 0
   }
  },
  "thresholds": [
   "0.00",
   "0.05",
   "0.10",
   "0.15",
   "0.20",
   "0.25",
   "0.30",
   "0.35",
   "0.40",
   "0.45",
   "0.50",
   "0.55",
   "0.60",
   "0.65",
   "0.70",
   "0.75",
   "0.80",
   "0.85",
   "0.90",
   "0.95",
   "1.00",
   "always safe"
  ]
 },
 "reasoning": [
  {
   "extra_cost": 6.306051658857413,
   "extra_time_median": 5.1255,
   "high": "Claude Sonnet 5.5 (high)",
   "high_setting": "high",
   "low": "Claude Sonnet 5.5 (low)",
   "low_setting": "low",
   "model": "Claude Sonnet 5.5",
   "reading": "no clear difference in league play",
   "risk": {
    "change": 4.671013027001997,
    "overlap": false
   },
   "sdt": {
    "main": {
     "caution_change": -0.21223824086791088,
     "dprime_change": 0.09634056399773128
    },
    "robust": {
     "caution_change": -0.19017251696227866,
     "dprime_change": 0.1524891153374457
    }
   }
  },
  {
   "extra_cost": 4.624952696284031,
   "extra_time_median": 21.486000000000004,
   "high": "DeepSeek-V4.1-Flash 1017 (low)",
   "high_setting": "low",
   "low": "DeepSeek-V4.1-Flash 1017 (off)",
   "low_setting": "off",
   "model": "DeepSeek-V4.1-Flash 1017",
   "reading": "lower error rate in league play",
   "risk": {
    "change": 9.96377461885507,
    "overlap": false
   },
   "sdt": {
    "main": {
     "caution_change": -0.10626281714288244,
     "dprime_change": 0.6355790877865484
    },
    "robust": {
     "caution_change": -0.17229186868355673,
     "dprime_change": 0.5939025087820184
    }
   }
  },
  {
   "extra_cost": 18.229822469210752,
   "extra_time_median": 7.4305,
   "high": "Gemini 3.8 Flash (high)",
   "high_setting": "high",
   "low": "Gemini 3.8 Flash (low)",
   "low_setting": "low",
   "model": "Gemini 3.8 Flash",
   "reading": "no clear difference in league play",
   "risk": {
    "change": 1.7194588174994578,
    "overlap": true
   },
   "sdt": {
    "main": {
     "caution_change": 0.06209340699293309,
     "dprime_change": 0.2098861904283542
    },
    "robust": {
     "caution_change": -0.013096300989216547,
     "dprime_change": 0.04754146498674383
    }
   }
  },
  {
   "extra_cost": 26.72809753419434,
   "extra_time_median": 120.2875,
   "high": "Gemma 4 31B (on)",
   "high_setting": "on",
   "low": "Gemma 4 31B (off)",
   "low_setting": "off",
   "model": "Gemma 4 31B",
   "reading": "no clear difference in league play",
   "risk": {
    "change": 1.8431934410737938,
    "overlap": true
   },
   "sdt": {
    "main": {
     "caution_change": 0.32711647295077495,
     "dprime_change": 0.3154900012008337
    },
    "robust": {
     "caution_change": 0.30736344597929055,
     "dprime_change": 0.32815497673128213
    }
   }
  }
 ],
 "references": {
  "Random (computed)": {
   "label": "computed: every legal play equally likely",
   "name": "Random (computed)",
   "positions": 39488,
   "reference": "Random",
   "risk": {
    "main": {
     "by_call": {
      "missed": 5.404547859524397,
      "needless": 57.30934713788334,
      "none": 8.040766442869675,
      "right_risk": 3.229444753320445,
      "right_safety": 3.9091815132573458,
      "wrong_risk": 40.025180479048885
     },
     "calls": {
      "missed": 2193.0993159416416,
      "needless": 11481.043980738523,
      "right_risk": 3300.253802962741,
      "right_safety": 4654.956019265889,
      "wrong_risk": 7646.646881095751
     },
     "confidence": null,
     "error_rate": 117.91846818590409,
     "found": {
      "den": 13140.0,
      "n": 3300.253802962741,
      "share": 0.2511608678053836
     },
     "good": {
      "den": 22427.944664797014,
      "n": 3300.253802962741,
      "share": 0.1471491860840388
     },
     "judgement": 26.982175824781727,
     "judgement_hi": 27.245920030450332,
     "judgement_lo": 26.729136960721082,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.7771183044739556,
      "x": 0.7488391321946266,
      "y": 0.7115173513100225
     },
     "margin": 0.001,
     "missed": {
      "den": 13140.0,
      "n": 2193.0993159416416,
      "share": 0.16690253545979009
     },
     "needless": {
      "den": 16136.0,
      "n": 11481.043980738523,
      "share": 0.7115173513100225
     },
     "needless_by_loss": {
      "big": {
       "den": 16136.0,
       "n": 10184.780866007348,
       "share": 0.6311837423157751
      },
      "medium": {
       "den": 16136.0,
       "n": 1021.0880636327779,
       "share": 0.06328012293212555
      },
      "small": {
       "den": 16136.0,
       "n": 275.17505109710294,
       "share": 0.01705348606204158
      }
     },
     "no_trade_off": 10212.0,
     "reference": "Random",
     "right_safety": {
      "den": 16136.0,
      "n": 4654.956019265889,
      "share": 0.2884826486902509
     },
     "risk_pays": 13140.0,
     "safety_pays": 16136.0,
     "scored": 39488,
     "sdt": {
      "aim": {
       "den": 10946.900684058492,
       "n": 3300.253802962741,
       "share": 0.30147837257432697
      },
      "beta": 2.487387988454754,
      "caution": -0.7620805780256942,
      "caution_hi": -0.7530752360939553,
      "caution_lo": -0.7710325048379066,
      "cstar": 2.230178377806836,
      "cstar_note": null,
      "cstar_status": "valid",
      "dprime": 0.40859205180076974,
      "dprime_hi": 0.4243048738931311,
      "dprime_lo": 0.3937061193934986,
      "false_alarm": {
       "den": 16136.0,
       "n": 11481.043980738523,
       "share": 0.7115173513100225
      },
      "false_alarm_rate": 0.711504243709396,
      "hit": {
       "den": 13140.0,
       "n": 10946.900684058492,
       "share": 0.83309746454022
      },
      "hit_rate": 0.833072116586142,
      "missed_loss": 0.19462391358712033,
      "missed_n": 2193.0993159416416,
      "na": null,
      "needless_loss": 0.39422050879299336,
      "needless_n": 11481.043980738523
     },
     "size": {
      "bad": {
       "mean": 17.788852799351197,
       "n": 19127.690861829455,
       "too_few": false
      },
      "right": {
       "mean": 15.362982711607641,
       "n": 3300.253802962741,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 7127.0,
        "n": 2248.880178710472,
        "share": 0.31554373210473863
       },
       "missed": {
        "den": 7127.0,
        "n": 1095.6487104353575,
        "share": 0.15373210473345833
       },
       "risk_pays": 7127.0,
       "wrong": {
        "den": 7127.0,
        "n": 3782.4711108535157,
        "share": 0.5307241631617112
       }
      },
      "medium": {
       "found": {
        "den": 4329.0,
        "n": 836.5500468692684,
        "share": 0.19324325407005508
       },
       "missed": {
        "den": 4329.0,
        "n": 751.3310958463874,
        "share": 0.1735576566981722
       },
       "risk_pays": 4329.0,
       "wrong": {
        "den": 4329.0,
        "n": 2741.118857283949,
        "share": 0.6331990892316814
       }
      },
      "small": {
       "found": {
        "den": 1684.0,
        "n": 214.82357738299862,
        "share": 0.12756744500178066
       },
       "missed": {
        "den": 1684.0,
        "n": 346.11950966001814,
        "share": 0.20553415062946445
       },
       "risk_pays": 1684.0,
       "wrong": {
        "den": 1684.0,
        "n": 1123.056912956922,
        "share": 0.6668984043687185
       }
      }
     },
     "trade_off": 29276.0,
     "wrong": {
      "den": 13140.0,
      "n": 7646.646881095751,
      "share": 0.5819365967348364
     }
    },
    "robust": {
     "by_call": {
      "missed": 5.147839956926229,
      "needless": 61.8679844028968,
      "none": 8.040766442869675,
      "right_risk": 2.6172836003554982,
      "right_safety": 4.1658894158555135,
      "wrong_risk": 36.07870436700119
     },
     "calls": {
      "missed": 1846.9798062816108,
      "needless": 12818.92447107956,
      "right_risk": 2710.351929773162,
      "right_safety": 5001.075528925821,
      "wrong_risk": 6898.668263945385
     },
     "confidence": null,
     "error_rate": 117.91846818590491,
     "found": {
      "den": 11456.0,
      "n": 2710.351929773162,
      "share": 0.23658798269667963
     },
     "good": {
      "den": 22427.944664798106,
      "n": 2710.351929773162,
      "share": 0.12084709367181597
     },
     "judgement": 25.861597588610135,
     "judgement_hi": 26.12904636139488,
     "judgement_lo": 25.598430774000107,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.7888115561647941,
      "x": 0.7634120173033341,
      "y": 0.7193560309247788
     },
     "margin": 0.02,
     "missed": {
      "den": 11456.0,
      "n": 1846.9798062816108,
      "share": 0.1612237959393864
     },
     "needless": {
      "den": 17820.0,
      "n": 12818.92447107956,
      "share": 0.7193560309247788
     },
     "needless_by_loss": {
      "big": {
       "den": 17820.0,
       "n": 11082.216195059831,
       "share": 0.6218976540437615
      },
      "medium": {
       "den": 17820.0,
       "n": 1205.8857968883426,
       "share": 0.0676703589724098
      },
      "small": {
       "den": 17820.0,
       "n": 530.8224791298994,
       "share": 0.029788017908524096
      }
     },
     "no_trade_off": 10212.0,
     "reference": "Random",
     "right_safety": {
      "den": 17820.0,
      "n": 5001.075528925821,
      "share": 0.2806439690755231
     },
     "risk_pays": 11456.0,
     "safety_pays": 17820.0,
     "scored": 39488,
     "sdt": {
      "aim": {
       "den": 9609.020193718547,
       "n": 2710.351929773162,
       "share": 0.28206329835220134
      },
      "beta": 2.6935570851600534,
      "caution": -0.7851063791082442,
      "caution_hi": -0.7760609545157103,
      "caution_lo": -0.794557751017933,
      "cstar": 2.4260476120833365,
      "cstar_note": null,
      "cstar_status": "valid",
      "dprime": 0.40842671497970195,
      "dprime_hi": 0.42575037278476546,
      "dprime_lo": 0.391857458919231,
      "false_alarm": {
       "den": 17820.0,
       "n": 12818.92447107956,
       "share": 0.7193560309247788
      },
      "false_alarm_rate": 0.7193437220739329,
      "hit": {
       "den": 11456.0,
       "n": 9609.020193718547,
       "share": 0.8387762040606274
      },
      "hit_rate": 0.8387466346965651,
      "missed_loss": 0.220119249303919,
      "missed_n": 1846.9798062816108,
      "na": null,
      "needless_loss": 0.38116192565347806,
      "needless_n": 12818.92447107956
     },
     "size": {
      "bad": {
       "mean": 17.70437338334238,
       "n": 19717.592735016347,
       "too_few": false
      },
      "right": {
       "mean": 15.449578628522294,
       "n": 2710.351929773162,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 7127.0,
        "n": 2046.0162705943599,
        "share": 0.28707959458318505
       },
       "missed": {
        "den": 7127.0,
        "n": 1095.6487104353575,
        "share": 0.15373210473345833
       },
       "risk_pays": 7127.0,
       "wrong": {
        "den": 7127.0,
        "n": 3985.3350189696116,
        "share": 0.5591883006832624
       }
      },
      "medium": {
       "found": {
        "den": 4329.0,
        "n": 664.3356591787397,
        "share": 0.1534616907319796
       },
       "missed": {
        "den": 4329.0,
        "n": 751.3310958463874,
        "share": 0.1735576566981722
       },
       "risk_pays": 4329.0,
       "wrong": {
        "den": 4329.0,
        "n": 2913.333244974311,
        "share": 0.6729806525697184
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 29276.0,
     "wrong": {
      "den": 11456.0,
      "n": 6898.668263945385,
      "share": 0.6021882213639477
     }
    }
   },
   "slug": "random-computed"
  },
  "Safest (computed)": {
   "label": "computed: always the best safe play",
   "name": "Safest (computed)",
   "positions": 39488,
   "reference": "Safest",
   "risk": {
    "main": {
     "by_call": {
      "missed": 25.22650396759279,
      "needless": 0.0,
      "none": 0.0,
      "right_risk": 0.0,
      "right_safety": 0.0005029549450889785,
      "wrong_risk": 0.0
     },
     "calls": {
      "missed": 13140.0,
      "needless": 0.0,
      "right_risk": 0.0,
      "right_safety": 16136.0,
      "wrong_risk": 0.0
     },
     "confidence": null,
     "error_rate": 25.227006922537882,
     "found": {
      "den": 13140.0,
      "n": 0.0,
      "share": 0.0
     },
     "good": {
      "den": 0.0,
      "n": 0.0,
      "share": null
     },
     "judgement": 50.0,
     "judgement_hi": 50.0,
     "judgement_lo": 50.0,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.0,
      "x": 1.0,
      "y": 0.0
     },
     "margin": 0.001,
     "missed": {
      "den": 13140.0,
      "n": 13140.0,
      "share": 1.0
     },
     "needless": {
      "den": 16136.0,
      "n": 0.0,
      "share": 0.0
     },
     "needless_by_loss": {
      "big": {
       "den": 16136.0,
       "n": 0.0,
       "share": 0.0
      },
      "medium": {
       "den": 16136.0,
       "n": 0.0,
       "share": 0.0
      },
      "small": {
       "den": 16136.0,
       "n": 0.0,
       "share": 0.0
      }
     },
     "no_trade_off": 10212.0,
     "reference": "Safest",
     "right_safety": {
      "den": 16136.0,
      "n": 16136.0,
      "share": 1.0
     },
     "risk_pays": 13140.0,
     "safety_pays": 16136.0,
     "scored": 39488,
     "sdt": {
      "aim": {
       "den": 0.0,
       "n": 0.0,
       "share": null
      },
      "beta": null,
      "caution": 3.980775999204851,
      "caution_hi": 3.984713793814678,
      "caution_lo": 3.9766131704243586,
      "cstar": null,
      "cstar_note": "N/A: no needless risks to price",
      "cstar_status": "na",
      "dprime": 0.04881476644767746,
      "dprime_hi": 0.05605428062420575,
      "dprime_lo": 0.041919615815803724,
      "false_alarm": {
       "den": 16136.0,
       "n": 0.0,
       "share": 0.0
      },
      "false_alarm_rate": 3.098469356138068e-05,
      "hit": {
       "den": 13140.0,
       "n": 0.0,
       "share": 0.0
      },
      "hit_rate": 3.804885472947264e-05,
      "missed_loss": 0.1516201200414466,
      "missed_n": 13140.0,
      "na": null,
      "needless_loss": null,
      "needless_n": 0.0
     },
     "size": {
      "bad": {
       "mean": null,
       "n": 0.0,
       "too_few": true
      },
      "right": {
       "mean": null,
       "n": 0.0,
       "too_few": true
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 7127.0,
        "n": 0.0,
        "share": 0.0
       },
       "missed": {
        "den": 7127.0,
        "n": 7127.0,
        "share": 1.0
       },
       "risk_pays": 7127.0,
       "wrong": {
        "den": 7127.0,
        "n": 0.0,
        "share": 0.0
       }
      },
      "medium": {
       "found": {
        "den": 4329.0,
        "n": 0.0,
        "share": 0.0
       },
       "missed": {
        "den": 4329.0,
        "n": 4329.0,
        "share": 1.0
       },
       "risk_pays": 4329.0,
       "wrong": {
        "den": 4329.0,
        "n": 0.0,
        "share": 0.0
       }
      },
      "small": {
       "found": {
        "den": 1684.0,
        "n": 0.0,
        "share": 0.0
       },
       "missed": {
        "den": 1684.0,
        "n": 1684.0,
        "share": 1.0
       },
       "risk_pays": 1684.0,
       "wrong": {
        "den": 1684.0,
        "n": 0.0,
        "share": 0.0
       }
      }
     },
     "trade_off": 29276.0,
     "wrong": {
      "den": 13140.0,
      "n": 0.0,
      "share": 0.0
     }
    },
    "robust": {
     "by_call": {
      "missed": 25.00500237437892,
      "needless": 0.0,
      "none": 0.0,
      "right_risk": 0.0,
      "right_safety": 0.22200454815896153,
      "wrong_risk": 0.0
     },
     "calls": {
      "missed": 11456.0,
      "needless": 0.0,
      "right_risk": 0.0,
      "right_safety": 17820.0,
      "wrong_risk": 0.0
     },
     "confidence": null,
     "error_rate": 25.227006922537882,
     "found": {
      "den": 11456.0,
      "n": 0.0,
      "share": 0.0
     },
     "good": {
      "den": 0.0,
      "n": 0.0,
      "share": null
     },
     "judgement": 50.0,
     "judgement_hi": 50.0,
     "judgement_lo": 50.0,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.0,
      "x": 1.0,
      "y": 0.0
     },
     "margin": 0.02,
     "missed": {
      "den": 11456.0,
      "n": 11456.0,
      "share": 1.0
     },
     "needless": {
      "den": 17820.0,
      "n": 0.0,
      "share": 0.0
     },
     "needless_by_loss": {
      "big": {
       "den": 17820.0,
       "n": 0.0,
       "share": 0.0
      },
      "medium": {
       "den": 17820.0,
       "n": 0.0,
       "share": 0.0
      },
      "small": {
       "den": 17820.0,
       "n": 0.0,
       "share": 0.0
      }
     },
     "no_trade_off": 10212.0,
     "reference": "Safest",
     "right_safety": {
      "den": 17820.0,
      "n": 17820.0,
      "share": 1.0
     },
     "risk_pays": 11456.0,
     "safety_pays": 17820.0,
     "scored": 39488,
     "sdt": {
      "aim": {
       "den": 0.0,
       "n": 0.0,
       "share": null
      },
      "beta": null,
      "caution": 3.9760271590154557,
      "caution_hi": 3.9802129866243163,
      "caution_lo": 3.971732517936577,
      "cstar": null,
      "cstar_note": "N/A: no needless risks to price",
      "cstar_status": "na",
      "dprime": 0.105115671557372,
      "dprime_hi": 0.1127373063025925,
      "dprime_lo": 0.09804353697497556,
      "false_alarm": {
       "den": 17820.0,
       "n": 0.0,
       "share": 0.0
      },
      "false_alarm_rate": 2.805678693676e-05,
      "hit": {
       "den": 11456.0,
       "n": 0.0,
       "share": 0.0
      },
      "hit_rate": 4.3641441913240814e-05,
      "missed_loss": 0.1723808543574502,
      "missed_n": 11456.0,
      "na": null,
      "needless_loss": null,
      "needless_n": 0.0
     },
     "size": {
      "bad": {
       "mean": null,
       "n": 0.0,
       "too_few": true
      },
      "right": {
       "mean": null,
       "n": 0.0,
       "too_few": true
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 7127.0,
        "n": 0.0,
        "share": 0.0
       },
       "missed": {
        "den": 7127.0,
        "n": 7127.0,
        "share": 1.0
       },
       "risk_pays": 7127.0,
       "wrong": {
        "den": 7127.0,
        "n": 0.0,
        "share": 0.0
       }
      },
      "medium": {
       "found": {
        "den": 4329.0,
        "n": 0.0,
        "share": 0.0
       },
       "missed": {
        "den": 4329.0,
        "n": 4329.0,
        "share": 1.0
       },
       "risk_pays": 4329.0,
       "wrong": {
        "den": 4329.0,
        "n": 0.0,
        "share": 0.0
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 29276.0,
     "wrong": {
      "den": 11456.0,
      "n": 0.0,
      "share": 0.0
     }
    }
   },
   "slug": "safest-computed"
  }
 },
 "scored": 39538,
 "setups": {
  "Claude Fable 5.1 (low)": {
   "close": {
    "decisions": 2008,
    "definition": "A decision is a close choice when the second most likely play has at least half the probability of the first (internally: split).",
    "eligible": 0,
    "exclusions": {
     "no_probabilities": 1811,
     "not_audited": 197
    },
    "label": "Probabilities over plays, not sampled reasoning.",
    "na": "N/A: no full probability distribution"
   },
   "confidence": null,
   "quality": {
    "best_share": 0.5866533864541833,
    "blunder_share": 0.1508964143426295,
    "clusters": 98,
    "cost_coverage": 1.0,
    "decisions": 2008,
    "engine_scored": 2008,
    "error_hi": 20.08414600740365,
    "error_lo": 15.912546035687877,
    "error_rate": 17.91991719389817,
    "first_date": "2026-10-20",
    "group_sizes": [
     14,
     15,
     16,
     17,
     18,
     19,
     20,
     21,
     22,
     23,
     24,
     25,
     26
    ],
    "grouped": 232,
    "invalid": 0,
    "last_date": "2026-10-26",
    "over_budget": 0.010956175298804782,
    "p95_loss": 0.18543246090412127,
    "probability_source": "choice_only",
    "retry_mean": 0.04436802788844621,
    "scored": 2008,
    "time_median": 1.545,
    "time_p90": 6.3351999999999995,
    "tokens_in": 5727.832669322709,
    "tokens_out": 106.68725099601593,
    "too_few": false,
    "usd_per_1000": 62.61268924302789,
    "worst_mean": 0.312866849899292
   },
   "risk": {
    "main": {
     "by_call": {
      "missed": 8.816372528969055,
      "needless": 1.5887150548368811,
      "none": 1.6067082663456282,
      "right_risk": 1.7387507431416873,
      "right_safety": 2.6834758659045534,
      "wrong_risk": 1.4858947347003626
     },
     "calls": {
      "missed": 354.0,
      "needless": 45.0,
      "right_risk": 270.0,
      "right_safety": 781.0,
      "wrong_risk": 34.0
     },
     "confidence": null,
     "error_rate": 17.91991719389817,
     "found": {
      "den": 658.0,
      "n": 270.0,
      "share": 0.41033434650455924
     },
     "good": {
      "den": 349.0,
      "n": 270.0,
      "share": 0.7736389684813754
     },
     "judgement": 67.79274638091803,
     "judgement_hi": 69.8806632715162,
     "judgement_lo": 65.56214804039983,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.08762886597938144,
      "x": 0.5896656534954408,
      "y": 0.05447941888619855
     },
     "margin": 0.001,
     "missed": {
      "den": 658.0,
      "n": 354.0,
      "share": 0.5379939209726444
     },
     "needless": {
      "den": 826.0,
      "n": 45.0,
      "share": 0.05447941888619855
     },
     "needless_by_loss": {
      "big": {
       "den": 826.0,
       "n": 28.0,
       "share": 0.03389830508474576
      },
      "medium": {
       "den": 826.0,
       "n": 11.0,
       "share": 0.013317191283292978
      },
      "small": {
       "den": 826.0,
       "n": 6.0,
       "share": 0.007263922518159807
      }
     },
     "no_trade_off": 524.0,
     "reference": null,
     "right_safety": {
      "den": 826.0,
      "n": 781.0,
      "share": 0.9455205811138014
     },
     "risk_pays": 658.0,
     "safety_pays": 826.0,
     "scored": 2008,
     "sdt": {
      "aim": {
       "den": 304.0,
       "n": 270.0,
       "share": 0.8881578947368421
      },
      "beta": 1.7795122429355013,
      "caution": 0.8466329996188342,
      "caution_hi": 0.9413087196381701,
      "caution_lo": 0.7597015865110309,
      "cstar": 0.38351179170783245,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.5027942253461237,
      "dprime_hi": 1.7002745214936514,
      "dprime_lo": 1.3341320826303429,
      "false_alarm": {
       "den": 826.0,
       "n": 45.0,
       "share": 0.05447941888619855
      },
      "false_alarm_rate": 0.05501813784764208,
      "hit": {
       "den": 658.0,
       "n": 304.0,
       "share": 0.46200607902735563
      },
      "hit_rate": 0.4620637329286798,
      "missed_loss": 0.1000185086902252,
      "missed_n": 354.0,
      "na": null,
      "needless_loss": 0.14178399244944254,
      "needless_n": 45.0
     },
     "size": {
      "bad": {
       "mean": 9.518987341772151,
       "n": 79.0,
       "too_few": true
      },
      "right": {
       "mean": 9.307407407407407,
       "n": 270.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 340.0,
        "n": 198.0,
        "share": 0.5823529411764706
       },
       "missed": {
        "den": 340.0,
        "n": 133.0,
        "share": 0.3911764705882353
       },
       "risk_pays": 340.0,
       "wrong": {
        "den": 340.0,
        "n": 9.0,
        "share": 0.026470588235294117
       }
      },
      "medium": {
       "found": {
        "den": 227.0,
        "n": 61.0,
        "share": 0.2687224669603524
       },
       "missed": {
        "den": 227.0,
        "n": 147.0,
        "share": 0.6475770925110133
       },
       "risk_pays": 227.0,
       "wrong": {
        "den": 227.0,
        "n": 19.0,
        "share": 0.08370044052863436
       }
      },
      "small": {
       "found": {
        "den": 91.0,
        "n": 11.0,
        "share": 0.12087912087912088
       },
       "missed": {
        "den": 91.0,
        "n": 74.0,
        "share": 0.8131868131868132
       },
       "risk_pays": 91.0,
       "wrong": {
        "den": 91.0,
        "n": 6.0,
        "share": 0.06593406593406594
       }
      }
     },
     "trade_off": 1484.0,
     "wrong": {
      "den": 658.0,
      "n": 34.0,
      "share": 0.05167173252279635
     }
    },
    "robust": {
     "by_call": {
      "missed": 8.367011449726455,
      "needless": 1.885454701000001,
      "none": 1.6067082663456282,
      "right_risk": 1.528716048752644,
      "right_safety": 3.1328369451471536,
      "wrong_risk": 1.3991897829262858
     },
     "calls": {
      "missed": 280.0,
      "needless": 62.0,
      "right_risk": 248.0,
      "right_safety": 855.0,
      "wrong_risk": 39.0
     },
     "confidence": null,
     "error_rate": 17.91991719389817,
     "found": {
      "den": 567.0,
      "n": 248.0,
      "share": 0.43738977072310403
     },
     "good": {
      "den": 349.0,
      "n": 248.0,
      "share": 0.7106017191977078
     },
     "judgement": 68.48889965938312,
     "judgement_hi": 70.79110400551546,
     "judgement_lo": 66.2742453656496,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.12225705329153605,
      "x": 0.562610229276896,
      "y": 0.06761177753544166
     },
     "margin": 0.02,
     "missed": {
      "den": 567.0,
      "n": 280.0,
      "share": 0.49382716049382713
     },
     "needless": {
      "den": 917.0,
      "n": 62.0,
      "share": 0.06761177753544166
     },
     "needless_by_loss": {
      "big": {
       "den": 917.0,
       "n": 32.0,
       "share": 0.03489640130861505
      },
      "medium": {
       "den": 917.0,
       "n": 13.0,
       "share": 0.014176663031624863
      },
      "small": {
       "den": 917.0,
       "n": 17.0,
       "share": 0.018538713195201745
      }
     },
     "no_trade_off": 524.0,
     "reference": null,
     "right_safety": {
      "den": 917.0,
      "n": 855.0,
      "share": 0.9323882224645583
     },
     "risk_pays": 567.0,
     "safety_pays": 917.0,
     "scored": 2008,
     "sdt": {
      "aim": {
       "den": 287.0,
       "n": 248.0,
       "share": 0.8641114982578397
      },
      "beta": 1.6458807205906643,
      "caution": 0.7373883700456119,
      "caution_hi": 0.829484024297705,
      "caution_lo": 0.6526273685013861,
      "cstar": 0.3309329362934881,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.5056695141409686,
      "dprime_hi": 1.6832876284851686,
      "dprime_lo": 1.3515600609373395,
      "false_alarm": {
       "den": 917.0,
       "n": 62.0,
       "share": 0.06761177753544166
      },
      "false_alarm_rate": 0.06808278867102396,
      "hit": {
       "den": 567.0,
       "n": 287.0,
       "share": 0.5061728395061729
      },
      "hit_rate": 0.5061619718309859,
      "missed_loss": 0.12000684993607658,
      "missed_n": 280.0,
      "na": null,
      "needless_loss": 0.12212880772929038,
      "needless_n": 62.0
     },
     "size": {
      "bad": {
       "mean": 9.455445544554456,
       "n": 101.0,
       "too_few": true
      },
      "right": {
       "mean": 9.314516129032258,
       "n": 248.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 340.0,
        "n": 194.0,
        "share": 0.5705882352941176
       },
       "missed": {
        "den": 340.0,
        "n": 133.0,
        "share": 0.3911764705882353
       },
       "risk_pays": 340.0,
       "wrong": {
        "den": 340.0,
        "n": 13.0,
        "share": 0.03823529411764706
       }
      },
      "medium": {
       "found": {
        "den": 227.0,
        "n": 54.0,
        "share": 0.23788546255506607
       },
       "missed": {
        "den": 227.0,
        "n": 147.0,
        "share": 0.6475770925110133
       },
       "risk_pays": 227.0,
       "wrong": {
        "den": 227.0,
        "n": 26.0,
        "share": 0.1145374449339207
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 1484.0,
     "wrong": {
      "den": 567.0,
      "n": 39.0,
      "share": 0.06878306878306878
     }
    }
   },
   "slug": "claude-fable-5-1-off"
  },
  "Claude Sonnet 5.5 (high)": {
   "close": {
    "decisions": 2164,
    "definition": "A decision is a close choice when the second most likely play has at least half the probability of the first (internally: split).",
    "eligible": 0,
    "exclusions": {
     "no_probabilities": 1991,
     "not_audited": 173
    },
    "label": "Probabilities over plays, not sampled reasoning.",
    "na": "N/A: no full probability distribution"
   },
   "confidence": null,
   "quality": {
    "best_share": 0.6252310536044362,
    "blunder_share": 0.13262476894639558,
    "clusters": 102,
    "cost_coverage": 1.0,
    "decisions": 2164,
    "engine_scored": 2164,
    "error_hi": 18.667323968785514,
    "error_lo": 14.880148234830248,
    "error_rate": 16.719389183519066,
    "first_date": "2026-10-19",
    "group_sizes": [
     14,
     15,
     16,
     17,
     18,
     19,
     20,
     21,
     22,
     23,
     24,
     25,
     26
    ],
    "grouped": 257,
    "invalid": 0,
    "last_date": "2026-10-26",
    "over_budget": 0.034195933456561925,
    "p95_loss": 0.18679431974887836,
    "probability_source": "choice_only",
    "retry_mean": 0.26092097966728284,
    "scored": 2164,
    "time_median": 6.6335,
    "time_p90": 17.642600000000005,
    "tokens_in": 5664.079020332717,
    "tokens_out": 851.862292051756,
    "too_few": false,
    "usd_per_1000": 19.846780961182994,
    "worst_mean": 0.3166428928573926
   },
   "risk": {
    "main": {
     "by_call": {
      "missed": 10.75532382385127,
      "needless": 1.1226311739182957,
      "none": 0.9225192178529644,
      "right_risk": 1.7010349773435187,
      "right_safety": 1.579108388058139,
      "wrong_risk": 0.6387716024948797
     },
     "calls": {
      "missed": 409.0,
      "needless": 48.0,
      "right_risk": 308.0,
      "right_safety": 841.0,
      "wrong_risk": 20.0
     },
     "confidence": null,
     "error_rate": 16.719389183519066,
     "found": {
      "den": 737.0,
      "n": 308.0,
      "share": 0.417910447761194
     },
     "good": {
      "den": 376.0,
      "n": 308.0,
      "share": 0.8191489361702128
     },
     "judgement": 68.19585984587746,
     "judgement_hi": 70.04433076294394,
     "judgement_lo": 66.31726630398019,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.046620046620046623,
      "x": 0.582089552238806,
      "y": 0.05399325084364454
     },
     "margin": 0.001,
     "missed": {
      "den": 737.0,
      "n": 409.0,
      "share": 0.5549525101763908
     },
     "needless": {
      "den": 889.0,
      "n": 48.0,
      "share": 0.05399325084364454
     },
     "needless_by_loss": {
      "big": {
       "den": 889.0,
       "n": 17.0,
       "share": 0.019122609673790775
      },
      "medium": {
       "den": 889.0,
       "n": 22.0,
       "share": 0.024746906636670417
      },
      "small": {
       "den": 889.0,
       "n": 9.0,
       "share": 0.010123734533183352
      }
     },
     "no_trade_off": 538.0,
     "reference": null,
     "right_safety": {
      "den": 889.0,
      "n": 841.0,
      "share": 0.9460067491563554
     },
     "risk_pays": 737.0,
     "safety_pays": 889.0,
     "scored": 2164,
     "sdt": {
      "aim": {
       "den": 328.0,
       "n": 308.0,
       "share": 0.9390243902439024
      },
      "beta": 1.0728276320303092,
      "caution": 0.8703752757510059,
      "caution_hi": 0.9530722512659652,
      "caution_lo": 0.8004081454375466,
      "cstar": 0.04799273587198055,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.4647593704021231,
      "dprime_hi": 1.6117284502965732,
      "dprime_lo": 1.330914404154321,
      "false_alarm": {
       "den": 889.0,
       "n": 48.0,
       "share": 0.05399325084364454
      },
      "false_alarm_rate": 0.05449438202247191,
      "hit": {
       "den": 737.0,
       "n": 328.0,
       "share": 0.4450474898236092
      },
      "hit_rate": 0.4451219512195122,
      "missed_loss": 0.11381183743185402,
      "missed_n": 409.0,
      "na": null,
      "needless_loss": 0.10122391084829967,
      "needless_n": 48.0
     },
     "size": {
      "bad": {
       "mean": 9.294117647058824,
       "n": 68.0,
       "too_few": true
      },
      "right": {
       "mean": 8.564935064935066,
       "n": 308.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 409.0,
        "n": 230.0,
        "share": 0.5623471882640587
       },
       "missed": {
        "den": 409.0,
        "n": 174.0,
        "share": 0.4254278728606357
       },
       "risk_pays": 409.0,
       "wrong": {
        "den": 409.0,
        "n": 5.0,
        "share": 0.012224938875305624
       }
      },
      "medium": {
       "found": {
        "den": 255.0,
        "n": 69.0,
        "share": 0.27058823529411763
       },
       "missed": {
        "den": 255.0,
        "n": 172.0,
        "share": 0.6745098039215687
       },
       "risk_pays": 255.0,
       "wrong": {
        "den": 255.0,
        "n": 14.0,
        "share": 0.054901960784313725
       }
      },
      "small": {
       "found": {
        "den": 73.0,
        "n": 9.0,
        "share": 0.1232876712328767
       },
       "missed": {
        "den": 73.0,
        "n": 63.0,
        "share": 0.863013698630137
       },
       "risk_pays": 73.0,
       "wrong": {
        "den": 73.0,
        "n": 1.0,
        "share": 0.0136986301369863
       }
      }
     },
     "trade_off": 1626.0,
     "wrong": {
      "den": 737.0,
      "n": 20.0,
      "share": 0.027137042062415198
     }
    },
    "robust": {
     "by_call": {
      "missed": 10.526062999877825,
      "needless": 1.1266203607516896,
      "none": 0.9225192178529644,
      "right_risk": 1.5629107294373503,
      "right_safety": 1.8083692120315848,
      "wrong_risk": 0.7729066635676541
     },
     "calls": {
      "missed": 346.0,
      "needless": 58.0,
      "right_risk": 292.0,
      "right_safety": 904.0,
      "wrong_risk": 26.0
     },
     "confidence": null,
     "error_rate": 16.719389183519066,
     "found": {
      "den": 664.0,
      "n": 292.0,
      "share": 0.4397590361445783
     },
     "good": {
      "den": 376.0,
      "n": 292.0,
      "share": 0.776595744680851
     },
     "judgement": 68.9733987926759,
     "judgement_hi": 71.09514880224592,
     "judgement_lo": 66.99550970653478,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.06989247311827956,
      "x": 0.5602409638554217,
      "y": 0.060291060291060294
     },
     "margin": 0.02,
     "missed": {
      "den": 664.0,
      "n": 346.0,
      "share": 0.5210843373493976
     },
     "needless": {
      "den": 962.0,
      "n": 58.0,
      "share": 0.060291060291060294
     },
     "needless_by_loss": {
      "big": {
       "den": 962.0,
       "n": 17.0,
       "share": 0.017671517671517672
      },
      "medium": {
       "den": 962.0,
       "n": 22.0,
       "share": 0.02286902286902287
      },
      "small": {
       "den": 962.0,
       "n": 19.0,
       "share": 0.01975051975051975
      }
     },
     "no_trade_off": 538.0,
     "reference": null,
     "right_safety": {
      "den": 962.0,
      "n": 904.0,
      "share": 0.9397089397089398
     },
     "risk_pays": 664.0,
     "safety_pays": 962.0,
     "scored": 2164,
     "sdt": {
      "aim": {
       "den": 318.0,
       "n": 292.0,
       "share": 0.9182389937106918
      },
      "beta": 0.9250532400424523,
      "caution": 0.8006616084262336,
      "caution_hi": 0.8905497166992971,
      "caution_lo": 0.7209532323750105,
      "cstar": -0.05208418983363,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.4957319403879545,
      "dprime_hi": 1.6504497696553848,
      "dprime_lo": 1.3676895689214128,
      "false_alarm": {
       "den": 962.0,
       "n": 58.0,
       "share": 0.060291060291060294
      },
      "false_alarm_rate": 0.06074766355140187,
      "hit": {
       "den": 664.0,
       "n": 318.0,
       "share": 0.4789156626506024
      },
      "hit_rate": 0.4789473684210526,
      "missed_loss": 0.13166705394066827,
      "missed_n": 346.0,
      "na": null,
      "needless_loss": 0.08406918829885023,
      "needless_n": 58.0
     },
     "size": {
      "bad": {
       "mean": 9.238095238095237,
       "n": 84.0,
       "too_few": true
      },
      "right": {
       "mean": 8.54109589041096,
       "n": 292.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 409.0,
        "n": 226.0,
        "share": 0.5525672371638142
       },
       "missed": {
        "den": 409.0,
        "n": 174.0,
        "share": 0.4254278728606357
       },
       "risk_pays": 409.0,
       "wrong": {
        "den": 409.0,
        "n": 9.0,
        "share": 0.022004889975550123
       }
      },
      "medium": {
       "found": {
        "den": 255.0,
        "n": 66.0,
        "share": 0.25882352941176473
       },
       "missed": {
        "den": 255.0,
        "n": 172.0,
        "share": 0.6745098039215687
       },
       "risk_pays": 255.0,
       "wrong": {
        "den": 255.0,
        "n": 17.0,
        "share": 0.06666666666666667
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 1626.0,
     "wrong": {
      "den": 664.0,
      "n": 26.0,
      "share": 0.0391566265060241
     }
    }
   },
   "slug": "claude-sonnet-5-5-high"
  },
  "Claude Sonnet 5.5 (low)": {
   "close": {
    "decisions": 2150,
    "definition": "A decision is a close choice when the second most likely play has at least half the probability of the first (internally: split).",
    "eligible": 0,
    "exclusions": {
     "no_probabilities": 1941,
     "not_audited": 209
    },
    "label": "Probabilities over plays, not sampled reasoning.",
    "na": "N/A: no full probability distribution"
   },
   "confidence": null,
   "quality": {
    "best_share": 0.6009302325581395,
    "blunder_share": 0.1572093023255814,
    "clusters": 106,
    "cost_coverage": 1.0,
    "decisions": 2150,
    "engine_scored": 2150,
    "error_hi": 22.40217673585783,
    "error_lo": 17.843091915644056,
    "error_rate": 20.14592355766962,
    "first_date": "2026-10-19",
    "group_sizes": [
     14,
     15,
     16,
     17,
     18,
     19,
     20,
     21,
     22,
     23,
     24,
     25,
     26
    ],
    "grouped": 217,
    "invalid": 0,
    "last_date": "2026-10-26",
    "over_budget": 0.0051162790697674414,
    "p95_loss": 0.21513046622276302,
    "probability_source": "choice_only",
    "retry_mean": 0.007716744186046512,
    "scored": 2150,
    "time_median": 1.508,
    "time_p90": 7.373700000000005,
    "tokens_in": 5578.8879069767445,
    "tokens_out": 238.2953488372093,
    "too_few": false,
    "usd_per_1000": 13.54072930232558,
    "worst_mean": 0.37866273163645353
   },
   "risk": {
    "main": {
     "by_call": {
      "missed": 12.956029495527577,
      "needless": 1.224625263103219,
      "none": 1.2643842641697374,
      "right_risk": 1.0717695466307706,
      "right_safety": 2.6683175009350446,
      "wrong_risk": 0.9607974873032681
     },
     "calls": {
      "missed": 433.0,
      "needless": 35.0,
      "right_risk": 204.0,
      "right_safety": 883.0,
      "wrong_risk": 24.0
     },
     "confidence": null,
     "error_rate": 20.14592355766962,
     "found": {
      "den": 661.0,
      "n": 204.0,
      "share": 0.3086232980332829
     },
     "good": {
      "den": 263.0,
      "n": 204.0,
      "share": 0.7756653992395437
     },
     "judgement": 63.52484681887547,
     "judgement_hi": 65.4427432194247,
     "judgement_lo": 61.58807674553018,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.0525164113785558,
      "x": 0.6913767019667171,
      "y": 0.03812636165577342
     },
     "margin": 0.001,
     "missed": {
      "den": 661.0,
      "n": 433.0,
      "share": 0.6550680786686838
     },
     "needless": {
      "den": 918.0,
      "n": 35.0,
      "share": 0.03812636165577342
     },
     "needless_by_loss": {
      "big": {
       "den": 918.0,
       "n": 25.0,
       "share": 0.027233115468409588
      },
      "medium": {
       "den": 918.0,
       "n": 6.0,
       "share": 0.006535947712418301
      },
      "small": {
       "den": 918.0,
       "n": 4.0,
       "share": 0.004357298474945534
      }
     },
     "no_trade_off": 571.0,
     "reference": null,
     "right_safety": {
      "den": 918.0,
      "n": 883.0,
      "share": 0.9618736383442266
     },
     "risk_pays": 661.0,
     "safety_pays": 918.0,
     "scored": 2150,
     "sdt": {
      "aim": {
       "den": 228.0,
       "n": 204.0,
       "share": 0.8947368421052632
      },
      "beta": 1.6240236897373985,
      "caution": 1.0826135166189168,
      "caution_hi": 1.1903185158354095,
      "caution_lo": 0.9884054400539627,
      "cstar": 0.3543555720225301,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.3684188064043918,
      "dprime_hi": 1.5826118010366297,
      "dprime_lo": 1.1878902056757485,
      "false_alarm": {
       "den": 918.0,
       "n": 35.0,
       "share": 0.03812636165577342
      },
      "false_alarm_rate": 0.03862894450489663,
      "hit": {
       "den": 661.0,
       "n": 228.0,
       "share": 0.34493192133131617
      },
      "hit_rate": 0.34516616314199394,
      "missed_loss": 0.12866264856990436,
      "missed_n": 433.0,
      "na": null,
      "needless_loss": 0.15045396089553834,
      "needless_n": 35.0
     },
     "size": {
      "bad": {
       "mean": 7.203389830508475,
       "n": 59.0,
       "too_few": true
      },
      "right": {
       "mean": 8.696078431372548,
       "n": 204.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 346.0,
        "n": 145.0,
        "share": 0.4190751445086705
       },
       "missed": {
        "den": 346.0,
        "n": 192.0,
        "share": 0.5549132947976878
       },
       "risk_pays": 346.0,
       "wrong": {
        "den": 346.0,
        "n": 9.0,
        "share": 0.02601156069364162
       }
      },
      "medium": {
       "found": {
        "den": 228.0,
        "n": 50.0,
        "share": 0.21929824561403508
       },
       "missed": {
        "den": 228.0,
        "n": 167.0,
        "share": 0.7324561403508771
       },
       "risk_pays": 228.0,
       "wrong": {
        "den": 228.0,
        "n": 11.0,
        "share": 0.04824561403508772
       }
      },
      "small": {
       "found": {
        "den": 87.0,
        "n": 9.0,
        "share": 0.10344827586206896
       },
       "missed": {
        "den": 87.0,
        "n": 74.0,
        "share": 0.8505747126436781
       },
       "risk_pays": 87.0,
       "wrong": {
        "den": 87.0,
        "n": 4.0,
        "share": 0.04597701149425287
       }
      }
     },
     "trade_off": 1579.0,
     "wrong": {
      "den": 661.0,
      "n": 24.0,
      "share": 0.036308623298033284
     }
    },
    "robust": {
     "by_call": {
      "missed": 12.6374822785688,
      "needless": 1.2794481876284578,
      "none": 1.2643842641697374,
      "right_risk": 0.9800809968349545,
      "right_safety": 2.9868647178938224,
      "wrong_risk": 0.9976631125738454
     },
     "calls": {
      "missed": 359.0,
      "needless": 48.0,
      "right_risk": 190.0,
      "right_safety": 957.0,
      "wrong_risk": 25.0
     },
     "confidence": null,
     "error_rate": 20.14592355766962,
     "found": {
      "den": 574.0,
      "n": 190.0,
      "share": 0.3310104529616725
     },
     "good": {
      "den": 263.0,
      "n": 190.0,
      "share": 0.7224334600760456
     },
     "judgement": 64.16246294659108,
     "judgement_hi": 66.19949013927358,
     "judgement_lo": 62.12608122330353,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.06510416666666667,
      "x": 0.6689895470383276,
      "y": 0.04776119402985075
     },
     "margin": 0.02,
     "missed": {
      "den": 574.0,
      "n": 359.0,
      "share": 0.6254355400696864
     },
     "needless": {
      "den": 1005.0,
      "n": 48.0,
      "share": 0.04776119402985075
     },
     "needless_by_loss": {
      "big": {
       "den": 1005.0,
       "n": 27.0,
       "share": 0.026865671641791045
      },
      "medium": {
       "den": 1005.0,
       "n": 7.0,
       "share": 0.006965174129353234
      },
      "small": {
       "den": 1005.0,
       "n": 14.0,
       "share": 0.013930348258706468
      }
     },
     "no_trade_off": 571.0,
     "reference": null,
     "right_safety": {
      "den": 1005.0,
      "n": 957.0,
      "share": 0.9522388059701492
     },
     "risk_pays": 574.0,
     "safety_pays": 1005.0,
     "scored": 2150,
     "sdt": {
      "aim": {
       "den": 215.0,
       "n": 190.0,
       "share": 0.8837209302325582
      },
      "beta": 1.325774081442745,
      "caution": 0.9908341253885122,
      "caution_hi": 1.0879567765181073,
      "caution_lo": 0.9108788552550569,
      "cstar": 0.20993709849264713,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.3432428250505088,
      "dprime_hi": 1.5317832372934626,
      "dprime_lo": 1.1762532630098654,
      "false_alarm": {
       "den": 1005.0,
       "n": 48.0,
       "share": 0.04776119402985075
      },
      "false_alarm_rate": 0.04821073558648111,
      "hit": {
       "den": 574.0,
       "n": 215.0,
       "share": 0.37456445993031356
      },
      "hit_rate": 0.37478260869565216,
      "missed_loss": 0.15136817213884635,
      "missed_n": 359.0,
      "na": null,
      "needless_loss": 0.11461723347504933,
      "needless_n": 48.0
     },
     "size": {
      "bad": {
       "mean": 7.602739726027397,
       "n": 73.0,
       "too_few": true
      },
      "right": {
       "mean": 8.652631578947368,
       "n": 190.0,
       "too_few": true
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 346.0,
        "n": 143.0,
        "share": 0.41329479768786126
       },
       "missed": {
        "den": 346.0,
        "n": 192.0,
        "share": 0.5549132947976878
       },
       "risk_pays": 346.0,
       "wrong": {
        "den": 346.0,
        "n": 11.0,
        "share": 0.031791907514450865
       }
      },
      "medium": {
       "found": {
        "den": 228.0,
        "n": 47.0,
        "share": 0.20614035087719298
       },
       "missed": {
        "den": 228.0,
        "n": 167.0,
        "share": 0.7324561403508771
       },
       "risk_pays": 228.0,
       "wrong": {
        "den": 228.0,
        "n": 14.0,
        "share": 0.06140350877192982
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 1579.0,
     "wrong": {
      "den": 574.0,
      "n": 25.0,
      "share": 0.04355400696864112
     }
    }
   },
   "slug": "claude-sonnet-5-5-low"
  },
  "Clef": {
   "close": {
    "close": {
     "den": 509,
     "n": 450,
     "share": 0.8840864440078585,
     "small": false
    },
    "decisions": 1978,
    "definition": "A decision is a close choice when the second most likely play has at least half the probability of the first (internally: split).",
    "eligible": 509,
    "exclusions": {
     "grouped": 3,
     "not_audited": 1466
    },
    "groups": {
     "close": {
      "by_count": [
       98,
       170,
       110,
       62,
       10
      ],
      "differ": {
       "hitting": {
        "den": 450,
        "n": 257,
        "share": 0.5711111111111111,
        "small": false
       },
       "points": {
        "den": 450,
        "n": 159,
        "share": 0.35333333333333333,
        "small": false
       },
       "prime": {
        "den": 450,
        "n": 128,
        "share": 0.28444444444444444,
        "small": false
       },
       "race": {
        "den": 450,
        "n": 72,
        "share": 0.16,
        "small": false
       }
      },
      "gap": {
       "mean": 0.1489286661810345,
       "median": 0.07253372669219971,
       "n": 450
      },
      "loss1": {
       "mean": 0.08080049289597405,
       "median": 0.011035770177841187,
       "n": 450
      },
      "loss2": {
       "mean": 0.15126674433549245,
       "median": 0.05922764539718628,
       "n": 450
      },
      "n": 450,
      "p1": {
       "mean": 0.2538227121347464,
       "median": 0.21887811218878112,
       "n": 450
      },
      "p2": {
       "mean": 0.1919007978977028,
       "median": 0.16369180581941806,
       "n": 450
      },
      "psum": {
       "mean": 0.4457235100324492,
       "median": 0.3839616038396161,
       "n": 450
      },
      "psum_bins": [
       6,
       42,
       109,
       79,
       60,
       44,
       37,
       32,
       9,
       32
      ]
     },
     "other": {
      "by_count": [
       4,
       22,
       22,
       11,
       0
      ],
      "differ": {
       "hitting": {
        "den": 59,
        "n": 39,
        "share": 0.6610169491525424,
        "small": true
       },
       "points": {
        "den": 59,
        "n": 28,
        "share": 0.4745762711864407,
        "small": true
       },
       "prime": {
        "den": 59,
        "n": 21,
        "share": 0.3559322033898305,
        "small": true
       },
       "race": {
        "den": 59,
        "n": 11,
        "share": 0.1864406779661017,
        "small": true
       }
      },
      "gap": {
       "mean": 0.17288835371954966,
       "median": 0.13484704494476318,
       "n": 59
      },
      "loss1": {
       "mean": 0.028933410422276644,
       "median": 0.0,
       "n": 59
      },
      "loss2": {
       "mean": 0.17352996387724148,
       "median": 0.132851243019104,
       "n": 59
      },
      "n": 59,
      "p1": {
       "mean": 0.43334571004055034,
       "median": 0.39527905581116224,
       "n": 59
      },
      "p2": {
       "mean": 0.15912222075171065,
       "median": 0.1414858514148585,
       "n": 59
      },
      "psum": {
       "mean": 0.592467930792261,
       "median": 0.5243524352435244,
       "n": 59
      },
      "psum_bins": [
       0,
       1,
       4,
       12,
       8,
       10,
       5,
       6,
       1,
       12
      ]
     }
    },
    "label": "Probabilities over plays, not sampled reasoning.",
    "margins": {
     "main": {
      "classes": {
       "none": {
        "close": {
         "den": 127,
         "n": 113,
         "share": 0.889763779527559,
         "small": true
        },
        "eligible": 127,
        "risk_close": {
         "den": 113,
         "n": 0,
         "share": 0.0,
         "small": true
        }
       },
       "risk": {
        "close": {
         "den": 174,
         "n": 159,
         "share": 0.9137931034482759,
         "small": true
        },
        "eligible": 174,
        "risk_close": {
         "den": 159,
         "n": 80,
         "share": 0.5031446540880503,
         "small": true
        }
       },
       "safety": {
        "close": {
         "den": 208,
         "n": 178,
         "share": 0.8557692307692307,
         "small": false
        },
        "eligible": 208,
        "risk_close": {
         "den": 178,
         "n": 107,
         "share": 0.601123595505618,
         "small": true
        }
       }
      },
      "margin": 0.001,
      "missed": {
       "adj_close": 0.5294117647058824,
       "adj_other": 0.4509803921568627,
       "adjusted": 0.07843137254901966,
       "adjusted_hi": 0.8333333333333334,
       "adjusted_left_out": 4,
       "adjusted_lo": -0.8,
       "close": {
        "den": 80,
        "n": 40,
        "share": 0.5,
        "small": true
       },
       "covered": {
        "den": 80,
        "n": 17,
        "share": 0.2125,
        "small": true
       },
       "crude": -0.16666666666666663,
       "crude_hi": 0.21669793621013134,
       "crude_left_out": 0,
       "crude_lo": -0.5283018867924528,
       "group": 89,
       "matches": 4,
       "na": null,
       "other": {
        "den": 9,
        "n": 6,
        "share": 0.6666666666666666,
        "small": true
       },
       "resamples": 2000,
       "strata": 23,
       "strata_both": 5
      },
      "stake": {
       "big": {
        "den": 107,
        "n": 99,
        "share": 0.9252336448598131,
        "small": true
       },
       "medium": {
        "den": 48,
        "n": 43,
        "share": 0.8958333333333334,
        "small": true
       },
       "small": {
        "den": 19,
        "n": 17,
        "share": 0.8947368421052632,
        "small": true
       }
      }
     },
     "robust": {
      "classes": {
       "none": {
        "close": {
         "den": 127,
         "n": 113,
         "share": 0.889763779527559,
         "small": true
        },
        "eligible": 127,
        "risk_close": {
         "den": 113,
         "n": 0,
         "share": 0.0,
         "small": true
        }
       },
       "risk": {
        "close": {
         "den": 155,
         "n": 142,
         "share": 0.9161290322580645,
         "small": true
        },
        "eligible": 155,
        "risk_close": {
         "den": 142,
         "n": 71,
         "share": 0.5,
         "small": true
        }
       },
       "safety": {
        "close": {
         "den": 227,
         "n": 195,
         "share": 0.8590308370044053,
         "small": false
        },
        "eligible": 227,
        "risk_close": {
         "den": 195,
         "n": 116,
         "share": 0.5948717948717949,
         "small": true
        }
       }
      },
      "margin": 0.02,
      "missed": {
       "adj_close": 0.4666666666666667,
       "adj_other": 0.4666666666666667,
       "adjusted": 0.0,
       "adjusted_hi": 0.8500000000000001,
       "adjusted_left_out": 9,
       "adjusted_lo": -1.0,
       "close": {
        "den": 71,
        "n": 33,
        "share": 0.4647887323943662,
        "small": true
       },
       "covered": {
        "den": 71,
        "n": 15,
        "share": 0.2112676056338028,
        "small": true
       },
       "crude": -0.2494969818913481,
       "crude_hi": 0.1701388888888889,
       "crude_left_out": 9,
       "crude_lo": -0.5862068965517242,
       "group": 78,
       "matches": 4,
       "na": null,
       "other": {
        "den": 7,
        "n": 5,
        "share": 0.7142857142857143,
        "small": true
       },
       "resamples": 2000,
       "strata": 23,
       "strata_both": 5
      },
      "stake": {
       "big": {
        "den": 107,
        "n": 99,
        "share": 0.9252336448598131,
        "small": true
       },
       "medium": {
        "den": 48,
        "n": 43,
        "share": 0.8958333333333334,
        "small": true
       },
       "small": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       }
      }
     }
    },
    "na": null,
    "risk_close": {
     "den": 450,
     "n": 187,
     "share": 0.41555555555555557,
     "small": false
    },
    "risk_close_of_eligible": {
     "den": 509,
     "n": 187,
     "share": 0.36738703339882123,
     "small": false
    },
    "risk_pair_other": {
     "den": 59,
     "n": 35,
     "share": 0.5932203389830508,
     "small": true
    },
    "ties": {
     "close": 4,
     "member": 2,
     "order": 2,
     "positive": 4,
     "top_two": 4,
     "zero": 0
    }
   },
   "confidence": {
    "auroc_best": 0.7112391757282743,
    "auroc_blunder": 0.6778940466544415,
    "confident_badly_wrong": {
     "den": 1956,
     "n": 0,
     "share": 0.0
    },
    "cutoffs": {
     "0.9": {
      "blunders": {
       "den": 0,
       "n": 0,
       "share": null
      },
      "kept": {
       "den": 1956,
       "n": 0,
       "share": 0.0
      },
      "mean_loss": null,
      "too_few": true
     },
     "0.99": {
      "blunders": {
       "den": 0,
       "n": 0,
       "share": null
      },
      "kept": {
       "den": 1956,
       "n": 0,
       "share": 0.0
      },
      "mean_loss": null,
      "too_few": true
     }
    },
    "ece": 0.1641242331288344,
    "held_out": {
     "blunders": 0.025624024829610587,
     "kept": 0.027560783097922818,
     "no_cutoff": 0,
     "runs": 300
    },
    "n": 1956,
    "surest": {
     "cut": 0.3827,
     "error_rate": 13.4626800967628,
     "n": 489,
     "too_few": false
    },
    "too_few": false
   },
   "quality": {
    "auroc": 0.7112391757282743,
    "best_share": 0.4448938321536906,
    "blunder_share": 0.27098078867542974,
    "brier": 0.2393501970552147,
    "clusters": 94,
    "confidence_n": 1956,
    "confident_badly_wrong": 0.0,
    "cost_coverage": 1.0,
    "decisions": 1978,
    "ece": 0.1641242331288344,
    "engine_scored": 1978,
    "error_hi": 39.589229467764994,
    "error_lo": 33.482122352187346,
    "error_rate": 36.536419001399445,
    "first_date": "2026-10-16",
    "group_sizes": [
     56,
     60,
     61,
     62,
     64,
     66,
     71,
     74,
     75,
     77,
     78,
     80,
     82,
     89,
     92,
     93,
     94,
     107
    ],
    "grouped": 22,
    "invalid": 0,
    "last_date": "2026-10-26",
    "mean_confidence": 0.2847510224948875,
    "over_budget": 0.0015166835187057635,
    "own_cost": {
     "decisions": 1978,
     "estimate": true,
     "excluded_hours": 243.23333333333332,
     "gaps": [
      [
       "2026-10-16T04:52Z",
       "2026-10-19T18:36Z",
       5144
      ],
      [
       "2026-10-19T18:43Z",
       "2026-10-19T20:53Z",
       129
      ],
      [
       "2026-10-19T21:57Z",
       "2026-10-20T05:03Z",
       426
      ],
      [
       "2026-10-20T05:26Z",
       "2026-10-26T09:05Z",
       8859
      ],
      [
       "2026-10-26T10:41Z",
       "2026-10-26T10:51Z",
       10
      ],
      [
       "2026-10-26T11:29Z",
       "2026-10-26T11:40Z",
       11
      ],
      [
       "2026-10-26T12:57Z",
       "2026-10-26T13:12Z",
       15
      ]
     ],
     "gpu_decisions": 1978,
     "gpu_usd": 11.34597609421184,
     "hosted_decisions": 0,
     "hosted_unpriced": 0,
     "hosted_usd": 0,
     "hours": 6.0104752777682435,
     "kind": "gpu",
     "parts": [
      {
       "gpus": 1,
       "hours": 1.5897311110628984,
       "machine": "dgx_spark",
       "usd": 0.33832311911552904,
       "usd_per_gpu_hour": 0.21281782608464228
      },
      {
       "gpus": 1,
       "hours": 4.4207441667053455,
       "machine": "rtx_pro_6000",
       "usd": 11.007652975096311,
       "usd_per_gpu_hour": 2.49
      }
     ],
     "provenance": "content/own_cost.json: wall clock on our machines × hourly rate",
     "spark_rate": {
      "chf_per_h": 0.17465558152207,
      "electricity_chf_per_h": 0.02244858,
      "purchase_chf_per_h": 0.15220700152207,
      "usd_per_h": 0.21281782608464228
     },
     "usd": 11.34597609421184,
     "usd_per_1000": 5.736084981906896
    },
    "p95_loss": 0.3336704969406126,
    "probability_source": "decision",
    "retry_mean": 2.1500176946410514,
    "scored": 1978,
    "time_median": 1.1324999999999998,
    "time_p90": 5.4129,
    "tokens_in": 4027.62891809909,
    "tokens_out": 0.0,
    "too_few": false,
    "usd_per_1000": 5.736084981906896,
    "very_confident": 0.0,
    "worst_mean": 0.530567927794023
   },
   "risk": {
    "main": {
     "by_call": {
      "missed": 11.85176096653191,
      "needless": 8.852436227010163,
      "none": 3.9174039485601613,
      "right_risk": 2.2574946645899656,
      "right_safety": 4.179025270398876,
      "wrong_risk": 5.478297924308372
     },
     "calls": {
      "missed": 301.0,
      "needless": 195.0,
      "right_risk": 263.0,
      "right_safety": 603.0,
      "wrong_risk": 102.0
     },
     "confidence": {
      "auroc_right_risk": 0.4879557763516808,
      "missed_mean_p": 0.22571952861952863,
      "missed_n": 297,
      "taken_grouped": 8,
      "taken_n": 552,
      "taken_other": 0
     },
     "error_rate": 36.536419001399445,
     "found": {
      "den": 666.0,
      "n": 263.0,
      "share": 0.3948948948948949
     },
     "good": {
      "den": 560.0,
      "n": 263.0,
      "share": 0.46964285714285714
     },
     "judgement": 57.526699631962785,
     "judgement_hi": 59.77403983905231,
     "judgement_lo": 55.405154965249224,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.2531017369727047,
      "x": 0.6051051051051051,
      "y": 0.24436090225563908
     },
     "margin": 0.001,
     "missed": {
      "den": 666.0,
      "n": 301.0,
      "share": 0.4519519519519519
     },
     "needless": {
      "den": 798.0,
      "n": 195.0,
      "share": 0.24436090225563908
     },
     "needless_by_loss": {
      "big": {
       "den": 798.0,
       "n": 123.0,
       "share": 0.15413533834586465
      },
      "medium": {
       "den": 798.0,
       "n": 56.0,
       "share": 0.07017543859649122
      },
      "small": {
       "den": 798.0,
       "n": 16.0,
       "share": 0.020050125313283207
      }
     },
     "no_trade_off": 514.0,
     "reference": null,
     "right_safety": {
      "den": 798.0,
      "n": 603.0,
      "share": 0.7556390977443609
     },
     "risk_pays": 666.0,
     "safety_pays": 798.0,
     "scored": 1978,
     "sdt": {
      "aim": {
       "den": 365.0,
       "n": 263.0,
       "share": 0.7205479452054795
      },
      "beta": 1.3814668546586857,
      "caution": 0.285387516268885,
      "caution_hi": 0.3674814382085515,
      "caution_lo": 0.20188786568165343,
      "cstar": 0.3980247750480277,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 0.811873765319813,
      "dprime_hi": 0.942451279563979,
      "dprime_lo": 0.6747317997894996,
      "false_alarm": {
       "den": 798.0,
       "n": 195.0,
       "share": 0.24436090225563908
      },
      "false_alarm_rate": 0.24468085106382978,
      "hit": {
       "den": 666.0,
       "n": 365.0,
       "share": 0.5480480480480481
      },
      "hit_rate": 0.547976011994003,
      "missed_loss": 0.1557660012744194,
      "missed_n": 301.0,
      "na": null,
      "needless_loss": 0.17959096263616514,
      "needless_n": 195.0
     },
     "size": {
      "bad": {
       "mean": 10.666666666666666,
       "n": 297.0,
       "too_few": false
      },
      "right": {
       "mean": 9.319391634980988,
       "n": 263.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 384.0,
        "n": 182.0,
        "share": 0.4739583333333333
       },
       "missed": {
        "den": 384.0,
        "n": 156.0,
        "share": 0.40625
       },
       "risk_pays": 384.0,
       "wrong": {
        "den": 384.0,
        "n": 46.0,
        "share": 0.11979166666666667
       }
      },
      "medium": {
       "found": {
        "den": 207.0,
        "n": 66.0,
        "share": 0.3188405797101449
       },
       "missed": {
        "den": 207.0,
        "n": 97.0,
        "share": 0.46859903381642515
       },
       "risk_pays": 207.0,
       "wrong": {
        "den": 207.0,
        "n": 44.0,
        "share": 0.21256038647342995
       }
      },
      "small": {
       "found": {
        "den": 75.0,
        "n": 15.0,
        "share": 0.2
       },
       "missed": {
        "den": 75.0,
        "n": 48.0,
        "share": 0.64
       },
       "risk_pays": 75.0,
       "wrong": {
        "den": 75.0,
        "n": 12.0,
        "share": 0.16
       }
      }
     },
     "trade_off": 1464.0,
     "wrong": {
      "den": 666.0,
      "n": 102.0,
      "share": 0.15315315315315314
     }
    },
    "robust": {
     "by_call": {
      "missed": 11.40943364710128,
      "needless": 9.44121383861295,
      "none": 3.9174039485601613,
      "right_risk": 2.04042866600054,
      "right_safety": 4.621352589829506,
      "wrong_risk": 5.106586311295011
     },
     "calls": {
      "missed": 253.0,
      "needless": 222.0,
      "right_risk": 240.0,
      "right_safety": 651.0,
      "wrong_risk": 98.0
     },
     "confidence": {
      "auroc_right_risk": 0.4996781806479296,
      "missed_mean_p": 0.2263819277108434,
      "missed_n": 249,
      "taken_grouped": 8,
      "taken_n": 552,
      "taken_other": 0
     },
     "error_rate": 36.536419001399445,
     "found": {
      "den": 591.0,
      "n": 240.0,
      "share": 0.40609137055837563
     },
     "good": {
      "den": 560.0,
      "n": 240.0,
      "share": 0.42857142857142855
     },
     "judgement": 57.58979189561637,
     "judgement_hi": 59.975785214366994,
     "judgement_lo": 55.2287223854449,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.2792022792022792,
      "x": 0.5939086294416244,
      "y": 0.2542955326460481
     },
     "margin": 0.02,
     "missed": {
      "den": 591.0,
      "n": 253.0,
      "share": 0.428087986463621
     },
     "needless": {
      "den": 873.0,
      "n": 222.0,
      "share": 0.2542955326460481
     },
     "needless_by_loss": {
      "big": {
       "den": 873.0,
       "n": 131.0,
       "share": 0.15005727376861397
      },
      "medium": {
       "den": 873.0,
       "n": 59.0,
       "share": 0.06758304696449026
      },
      "small": {
       "den": 873.0,
       "n": 32.0,
       "share": 0.03665521191294387
      }
     },
     "no_trade_off": 514.0,
     "reference": null,
     "right_safety": {
      "den": 873.0,
      "n": 651.0,
      "share": 0.7457044673539519
     },
     "risk_pays": 591.0,
     "safety_pays": 873.0,
     "scored": 1978,
     "sdt": {
      "aim": {
       "den": 338.0,
       "n": 240.0,
       "share": 0.7100591715976331
      },
      "beta": 1.39302226153952,
      "caution": 0.23961102265029194,
      "caution_hi": 0.32342275702741147,
      "caution_lo": 0.15025752853160723,
      "cstar": 0.3941019316426221,
      "cstar_note": null,
      "cstar_status": "valid",
      "dprime": 0.8410912230068482,
      "dprime_hi": 0.9864445983297268,
      "dprime_lo": 0.7034706578129915,
      "false_alarm": {
       "den": 873.0,
       "n": 222.0,
       "share": 0.2542955326460481
      },
      "false_alarm_rate": 0.2545766590389016,
      "hit": {
       "den": 591.0,
       "n": 338.0,
       "share": 0.571912013536379
      },
      "hit_rate": 0.5717905405405406,
      "missed_loss": 0.1784020533910382,
      "missed_n": 253.0,
      "na": null,
      "needless_loss": 0.1682407294844722,
      "needless_n": 222.0
     },
     "size": {
      "bad": {
       "mean": 10.7375,
       "n": 320.0,
       "too_few": false
      },
      "right": {
       "mean": 9.095833333333333,
       "n": 240.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 384.0,
        "n": 179.0,
        "share": 0.4661458333333333
       },
       "missed": {
        "den": 384.0,
        "n": 156.0,
        "share": 0.40625
       },
       "risk_pays": 384.0,
       "wrong": {
        "den": 384.0,
        "n": 49.0,
        "share": 0.12760416666666666
       }
      },
      "medium": {
       "found": {
        "den": 207.0,
        "n": 61.0,
        "share": 0.2946859903381642
       },
       "missed": {
        "den": 207.0,
        "n": 97.0,
        "share": 0.46859903381642515
       },
       "risk_pays": 207.0,
       "wrong": {
        "den": 207.0,
        "n": 49.0,
        "share": 0.23671497584541062
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 1464.0,
     "wrong": {
      "den": 591.0,
      "n": 98.0,
      "share": 0.1658206429780034
     }
    }
   },
   "slug": "clef"
  },
  "DeepSeek-V4.1-Flash (low)": {
   "close": {
    "close": {
     "den": 408,
     "n": 0,
     "share": 0.0,
     "small": false
    },
    "decisions": 2226,
    "definition": "A decision is a close choice when the second most likely play has at least half the probability of the first (internally: split).",
    "eligible": 408,
    "exclusions": {
     "grouped": 245,
     "invalid": 38,
     "not_audited": 265,
     "sparse": 1270
    },
    "groups": {
     "close": {
      "by_count": [
       0,
       0,
       0,
       0,
       0
      ],
      "differ": {
       "hitting": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       },
       "points": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       },
       "prime": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       },
       "race": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       }
      },
      "gap": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "loss1": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "loss2": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "n": 0,
      "p1": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "p2": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "psum": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "psum_bins": [
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0
      ]
     },
     "other": {
      "by_count": [
       122,
       170,
       99,
       16,
       1
      ],
      "differ": {
       "hitting": {
        "den": 408,
        "n": 178,
        "share": 0.4362745098039216,
        "small": false
       },
       "points": {
        "den": 408,
        "n": 60,
        "share": 0.14705882352941177,
        "small": false
       },
       "prime": {
        "den": 408,
        "n": 115,
        "share": 0.2818627450980392,
        "small": false
       },
       "race": {
        "den": 408,
        "n": 67,
        "share": 0.1642156862745098,
        "small": false
       }
      },
      "gap": {
       "mean": 0.15196647977127747,
       "median": 0.0653681755065918,
       "n": 408
      },
      "loss1": {
       "mean": 0.014378718593541314,
       "median": 0.0,
       "n": 408
      },
      "loss2": {
       "mean": 0.14854872343586942,
       "median": 0.05171555280685425,
       "n": 408
      },
      "n": 408,
      "p1": {
       "mean": 0.9999118874889622,
       "median": 0.9999940090536339,
       "n": 408
      },
      "p2": {
       "mean": 6.726746655709201e-05,
       "median": 4.785090162898317e-06,
       "n": 408
      },
      "psum": {
       "mean": 0.9999791549555194,
       "median": 0.9999999676040366,
       "n": 408
      },
      "psum_bins": [
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       408
      ]
     }
    },
    "label": "Probabilities over plays, not sampled reasoning.",
    "margins": {
     "main": {
      "classes": {
       "none": {
        "close": {
         "den": 204,
         "n": 0,
         "share": 0.0,
         "small": false
        },
        "eligible": 204,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       },
       "risk": {
        "close": {
         "den": 54,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 54,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       },
       "safety": {
        "close": {
         "den": 150,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 150,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       }
      },
      "margin": 0.001,
      "missed": {
       "close": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       },
       "group": 45,
       "na": "N/A: no close choices in this group",
       "other": {
        "den": 45,
        "n": 32,
        "share": 0.7111111111111111,
        "small": true
       }
      },
      "stake": {
       "big": {
        "den": 25,
        "n": 0,
        "share": 0.0,
        "small": true
       },
       "medium": {
        "den": 20,
        "n": 0,
        "share": 0.0,
        "small": true
       },
       "small": {
        "den": 9,
        "n": 0,
        "share": 0.0,
        "small": true
       }
      }
     },
     "robust": {
      "classes": {
       "none": {
        "close": {
         "den": 204,
         "n": 0,
         "share": 0.0,
         "small": false
        },
        "eligible": 204,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       },
       "risk": {
        "close": {
         "den": 45,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 45,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       },
       "safety": {
        "close": {
         "den": 159,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 159,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       }
      },
      "margin": 0.02,
      "missed": {
       "close": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       },
       "group": 36,
       "na": "N/A: no close choices in this group",
       "other": {
        "den": 36,
        "n": 23,
        "share": 0.6388888888888888,
        "small": true
       }
      },
      "stake": {
       "big": {
        "den": 25,
        "n": 0,
        "share": 0.0,
        "small": true
       },
       "medium": {
        "den": 20,
        "n": 0,
        "share": 0.0,
        "small": true
       },
       "small": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       }
      }
     }
    },
    "na": null,
    "risk_close": {
     "den": 0,
     "n": 0,
     "share": null,
     "small": false
    },
    "risk_close_of_eligible": {
     "den": 408,
     "n": 0,
     "share": 0.0,
     "small": false
    },
    "risk_pair_other": {
     "den": 408,
     "n": 168,
     "share": 0.4117647058823529,
     "small": false
    },
    "ties": {
     "close": 0,
     "member": 14,
     "order": 0,
     "positive": 14,
     "top_two": 14,
     "zero": 0
    }
   },
   "confidence": {
    "auroc_best": 0.49422956168612053,
    "auroc_blunder": 0.5103594456132149,
    "confident_badly_wrong": {
     "den": 1904,
     "n": 248,
     "share": 0.13025210084033614
    },
    "cutoffs": {
     "0.9": {
      "blunders": {
       "den": 1904,
       "n": 248,
       "share": 0.13025210084033614
      },
      "kept": {
       "den": 1904,
       "n": 1904,
       "share": 1.0
      },
      "mean_loss": 0.036728667110956015,
      "too_few": false
     },
     "0.99": {
      "blunders": {
       "den": 1903,
       "n": 248,
       "share": 0.1303205465055176
      },
      "kept": {
       "den": 1904,
       "n": 1903,
       "share": 0.9994747899159664
      },
      "mean_loss": 0.036738690550679354,
      "too_few": false
     }
    },
    "ece": 0.377556155836415,
    "held_out": {
     "blunders": 0.14388565325717786,
     "kept": 0.01266006596439424,
     "no_cutoff": 151,
     "runs": 300
    },
    "n": 1904,
    "surest": {
     "cut": 0.9999999377273622,
     "error_rate": 16.40700755499992,
     "n": 476,
     "too_few": false
    },
    "too_few": false
   },
   "quality": {
    "auroc": 0.49422956168612053,
    "best_share": 0.596892138939671,
    "blunder_share": 0.13893967093235832,
    "brier": 0.3775700661677277,
    "clusters": 102,
    "confidence_n": 1904,
    "confident_badly_wrong": 0.13025210084033614,
    "cost_coverage": 1.0,
    "decisions": 2226,
    "ece": 0.377556155836415,
    "engine_scored": 2226,
    "error_hi": 21.318701758752912,
    "error_lo": 16.596505586721516,
    "error_rate": 18.99183068894381,
    "first_date": "2026-10-19",
    "group_sizes": [
     14,
     15,
     16,
     17,
     18,
     19,
     20,
     21,
     22,
     23,
     24,
     25,
     26
    ],
    "grouped": 295,
    "invalid": 38,
    "last_date": "2026-10-26",
    "mean_confidence": 0.9999301054162469,
    "over_budget": 0.6371115173674589,
    "own_cost": {
     "decisions": 2226,
     "estimate": true,
     "excluded_hours": 33.3,
     "gaps": [
      [
       "2026-10-21T01:21Z",
       "2026-10-21T06:01Z",
       280
      ],
      [
       "2026-10-21T06:53Z",
       "2026-10-21T07:05Z",
       12
      ],
      [
       "2026-10-21T07:16Z",
       "2026-10-21T07:46Z",
       30
      ],
      [
       "2026-10-21T07:46Z",
       "2026-10-21T07:58Z",
       12
      ],
      [
       "2026-10-21T07:58Z",
       "2026-10-21T08:23Z",
       25
      ],
      [
       "2026-10-21T08:23Z",
       "2026-10-21T08:34Z",
       11
      ],
      [
       "2026-10-21T08:42Z",
       "2026-10-21T09:05Z",
       23
      ],
      [
       "2026-10-21T09:18Z",
       "2026-10-21T09:31Z",
       13
      ],
      [
       "2026-10-21T09:31Z",
       "2026-10-21T09:46Z",
       16
      ],
      [
       "2026-10-21T09:55Z",
       "2026-10-21T10:31Z",
       36
      ],
      [
       "2026-10-21T10:31Z",
       "2026-10-21T10:53Z",
       22
      ],
      [
       "2026-10-21T10:53Z",
       "2026-10-21T11:08Z",
       15
      ],
      [
       "2026-10-21T11:09Z",
       "2026-10-21T11:28Z",
       20
      ],
      [
       "2026-10-21T11:28Z",
       "2026-10-21T11:46Z",
       17
      ],
      [
       "2026-10-21T11:49Z",
       "2026-10-21T12:08Z",
       19
      ],
      [
       "2026-10-21T12:08Z",
       "2026-10-21T12:32Z",
       24
      ],
      [
       "2026-10-21T12:42Z",
       "2026-10-21T13:01Z",
       20
      ],
      [
       "2026-10-21T13:01Z",
       "2026-10-21T13:14Z",
       13
      ],
      [
       "2026-10-21T13:14Z",
       "2026-10-21T13:28Z",
       14
      ],
      [
       "2026-10-21T13:28Z",
       "2026-10-21T22:19Z",
       531
      ],
      [
       "2026-10-21T22:19Z",
       "2026-10-21T22:33Z",
       15
      ],
      [
       "2026-10-21T22:33Z",
       "2026-10-21T22:49Z",
       15
      ],
      [
       "2026-10-21T22:49Z",
       "2026-10-21T23:36Z",
       47
      ],
      [
       "2026-10-21T23:58Z",
       "2026-10-22T00:09Z",
       11
      ],
      [
       "2026-10-22T00:18Z",
       "2026-10-22T00:35Z",
       17
      ],
      [
       "2026-10-22T00:35Z",
       "2026-10-22T00:48Z",
       13
      ],
      [
       "2026-10-22T00:48Z",
       "2026-10-22T01:07Z",
       19
      ],
      [
       "2026-10-22T01:40Z",
       "2026-10-22T01:52Z",
       11
      ],
      [
       "2026-10-22T02:01Z",
       "2026-10-22T02:12Z",
       12
      ],
      [
       "2026-10-22T02:54Z",
       "2026-10-22T03:29Z",
       35
      ],
      [
       "2026-10-22T03:29Z",
       "2026-10-22T03:44Z",
       16
      ],
      [
       "2026-10-22T03:54Z",
       "2026-10-22T04:05Z",
       11
      ],
      [
       "2026-10-22T04:17Z",
       "2026-10-22T04:32Z",
       15
      ],
      [
       "2026-10-22T04:32Z",
       "2026-10-22T04:49Z",
       17
      ],
      [
       "2026-10-22T04:49Z",
       "2026-10-22T05:01Z",
       12
      ],
      [
       "2026-10-22T05:20Z",
       "2026-10-22T05:31Z",
       11
      ],
      [
       "2026-10-22T05:49Z",
       "2026-10-22T06:08Z",
       19
      ],
      [
       "2026-10-22T06:13Z",
       "2026-10-22T15:21Z",
       549
      ]
     ],
     "gpu_decisions": 670,
     "gpu_usd": 81.21095160051183,
     "hosted_decisions": 1556,
     "hosted_unpriced": 0,
     "hosted_usd": 23.6162946168,
     "hours": 8.153710000051388,
     "kind": "gpu",
     "parts": [
      {
       "gpus": 4,
       "hours": 8.153710000051388,
       "machine": "rtx_pro_6000",
       "usd": 81.21095160051183,
       "usd_per_gpu_hour": 2.49
      }
     ],
     "provenance": "content/own_cost.json: wall clock on our machines × hourly rate",
     "spark_rate": null,
     "usd": 104.82724621731184,
     "usd_per_1000": 47.09220405090379
    },
    "p95_loss": 0.1969535827636721,
    "probability_source": "logprobs",
    "retry_mean": 1.9875260557053012,
    "scored": 2188,
    "time_median": 43.79900000000001,
    "time_p90": 173.1085,
    "tokens_in": 4364.341868823001,
    "tokens_out": 12486.809074573226,
    "too_few": false,
    "usd_per_1000": 47.09220405090379,
    "very_confident": 1.0,
    "worst_mean": 0.37231273279277555
   },
   "risk": {
    "main": {
     "by_call": {
      "missed": 8.558630507432564,
      "needless": 2.271314098804481,
      "none": 0.9315957534901621,
      "right_risk": 2.4104529286869267,
      "right_safety": 2.8868416763528826,
      "wrong_risk": 1.9329957241767939
     },
     "calls": {
      "missed": 368.0,
      "needless": 67.0,
      "right_risk": 338.0,
      "right_safety": 891.0,
      "wrong_risk": 45.0
     },
     "confidence": {
      "auroc_right_risk": 0.5251569858712716,
      "missed_mean_p": 0.9999460625310377,
      "missed_n": 304,
      "taken_grouped": 79,
      "taken_n": 371,
      "taken_other": 0
     },
     "error_rate": 18.99183068894381,
     "found": {
      "den": 751.0,
      "n": 338.0,
      "share": 0.45006657789613846
     },
     "good": {
      "den": 450.0,
      "n": 338.0,
      "share": 0.7511111111111111
     },
     "judgement": 69.00646041881528,
     "judgement_hi": 70.86759721568757,
     "judgement_lo": 67.25173854173836,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.1089588377723971,
      "x": 0.5499334221038615,
      "y": 0.06993736951983298
     },
     "margin": 0.001,
     "missed": {
      "den": 751.0,
      "n": 368.0,
      "share": 0.49001331557922767
     },
     "needless": {
      "den": 958.0,
      "n": 67.0,
      "share": 0.06993736951983298
     },
     "needless_by_loss": {
      "big": {
       "den": 958.0,
       "n": 33.0,
       "share": 0.03444676409185804
      },
      "medium": {
       "den": 958.0,
       "n": 21.0,
       "share": 0.021920668058455117
      },
      "small": {
       "den": 958.0,
       "n": 13.0,
       "share": 0.013569937369519834
      }
     },
     "no_trade_off": 479.0,
     "reference": null,
     "right_safety": {
      "den": 958.0,
      "n": 891.0,
      "share": 0.930062630480167
     },
     "risk_pays": 751.0,
     "safety_pays": 958.0,
     "scored": 2188,
     "sdt": {
      "aim": {
       "den": 383.0,
       "n": 338.0,
       "share": 0.8825065274151436
      },
      "beta": 1.859394351501714,
      "caution": 0.723960690166419,
      "caution_hi": 0.8125396921545449,
      "caution_lo": 0.63347792066745,
      "cstar": 0.41407311856596424,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.4979258237678221,
      "dprime_hi": 1.642248415133445,
      "dprime_lo": 1.3601561962846926,
      "false_alarm": {
       "den": 958.0,
       "n": 67.0,
       "share": 0.06993736951983298
      },
      "false_alarm_rate": 0.07038581856100104,
      "hit": {
       "den": 751.0,
       "n": 383.0,
       "share": 0.5099866844207723
      },
      "hit_rate": 0.5099734042553191,
      "missed_loss": 0.10177328016446985,
      "missed_n": 368.0,
      "na": null,
      "needless_loss": 0.14834732084131952,
      "needless_n": 67.0
     },
     "size": {
      "bad": {
       "mean": 10.169642857142858,
       "n": 112.0,
       "too_few": true
      },
      "right": {
       "mean": 9.142011834319527,
       "n": 338.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 406.0,
        "n": 255.0,
        "share": 0.6280788177339901
       },
       "missed": {
        "den": 406.0,
        "n": 137.0,
        "share": 0.3374384236453202
       },
       "risk_pays": 406.0,
       "wrong": {
        "den": 406.0,
        "n": 14.0,
        "share": 0.034482758620689655
       }
      },
      "medium": {
       "found": {
        "den": 234.0,
        "n": 70.0,
        "share": 0.29914529914529914
       },
       "missed": {
        "den": 234.0,
        "n": 147.0,
        "share": 0.6282051282051282
       },
       "risk_pays": 234.0,
       "wrong": {
        "den": 234.0,
        "n": 17.0,
        "share": 0.07264957264957266
       }
      },
      "small": {
       "found": {
        "den": 111.0,
        "n": 13.0,
        "share": 0.11711711711711711
       },
       "missed": {
        "den": 111.0,
        "n": 84.0,
        "share": 0.7567567567567568
       },
       "risk_pays": 111.0,
       "wrong": {
        "den": 111.0,
        "n": 14.0,
        "share": 0.12612612612612611
       }
      }
     },
     "trade_off": 1709.0,
     "wrong": {
      "den": 751.0,
      "n": 45.0,
      "share": 0.05992010652463382
     }
    },
    "robust": {
     "by_call": {
      "missed": 8.232937768183417,
      "needless": 2.6745189260521145,
      "none": 0.9315957534901621,
      "right_risk": 2.236659909297169,
      "right_safety": 3.2125344156020303,
      "wrong_risk": 1.703583916318918
     },
     "calls": {
      "missed": 284.0,
      "needless": 94.0,
      "right_risk": 315.0,
      "right_safety": 975.0,
      "wrong_risk": 41.0
     },
     "confidence": {
      "auroc_right_risk": 0.5038940809968847,
      "missed_mean_p": 0.999939758656012,
      "missed_n": 237,
      "taken_grouped": 79,
      "taken_n": 371,
      "taken_other": 0
     },
     "error_rate": 18.99183068894381,
     "found": {
      "den": 640.0,
      "n": 315.0,
      "share": 0.4921875
     },
     "good": {
      "den": 450.0,
      "n": 315.0,
      "share": 0.7
     },
     "judgement": 70.21274263330216,
     "judgement_hi": 72.15847921584951,
     "judgement_lo": 68.24139533498973,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.12615384615384614,
      "x": 0.5078125,
      "y": 0.08793264733395698
     },
     "margin": 0.02,
     "missed": {
      "den": 640.0,
      "n": 284.0,
      "share": 0.44375
     },
     "needless": {
      "den": 1069.0,
      "n": 94.0,
      "share": 0.08793264733395698
     },
     "needless_by_loss": {
      "big": {
       "den": 1069.0,
       "n": 40.0,
       "share": 0.037418147801683815
      },
      "medium": {
       "den": 1069.0,
       "n": 27.0,
       "share": 0.025257249766136577
      },
      "small": {
       "den": 1069.0,
       "n": 27.0,
       "share": 0.025257249766136577
      }
     },
     "no_trade_off": 479.0,
     "reference": null,
     "right_safety": {
      "den": 1069.0,
      "n": 975.0,
      "share": 0.912067352666043
     },
     "risk_pays": 640.0,
     "safety_pays": 1069.0,
     "scored": 2188,
     "sdt": {
      "aim": {
       "den": 356.0,
       "n": 315.0,
       "share": 0.8848314606741573
      },
      "beta": 1.6393778408894508,
      "caution": 0.6049704735659669,
      "caution_hi": 0.6835237606730278,
      "caution_lo": 0.5260743734197818,
      "cstar": 0.33121536293134246,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.4924332016870827,
      "dprime_hi": 1.6311872575078856,
      "dprime_lo": 1.3624735511951631,
      "false_alarm": {
       "den": 1069.0,
       "n": 94.0,
       "share": 0.08793264733395698
      },
      "false_alarm_rate": 0.0883177570093458,
      "hit": {
       "den": 640.0,
       "n": 356.0,
       "share": 0.55625
      },
      "hit_rate": 0.5561622464898596,
      "missed_loss": 0.12685681575200927,
      "missed_n": 284.0,
      "na": null,
      "needless_loss": 0.1245073917064261,
      "needless_n": 94.0
     },
     "size": {
      "bad": {
       "mean": 10.214814814814815,
       "n": 135.0,
       "too_few": true
      },
      "right": {
       "mean": 9.047619047619047,
       "n": 315.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 406.0,
        "n": 253.0,
        "share": 0.6231527093596059
       },
       "missed": {
        "den": 406.0,
        "n": 137.0,
        "share": 0.3374384236453202
       },
       "risk_pays": 406.0,
       "wrong": {
        "den": 406.0,
        "n": 16.0,
        "share": 0.03940886699507389
       }
      },
      "medium": {
       "found": {
        "den": 234.0,
        "n": 62.0,
        "share": 0.26495726495726496
       },
       "missed": {
        "den": 234.0,
        "n": 147.0,
        "share": 0.6282051282051282
       },
       "risk_pays": 234.0,
       "wrong": {
        "den": 234.0,
        "n": 25.0,
        "share": 0.10683760683760683
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 1709.0,
     "wrong": {
      "den": 640.0,
      "n": 41.0,
      "share": 0.0640625
     }
    }
   },
   "slug": "deepseek-v4-1-flash-on"
  },
  "DeepSeek-V4.1-Flash 1017 (low)": {
   "close": {
    "close": {
     "den": 180,
     "n": 0,
     "share": 0.0,
     "small": true
    },
    "decisions": 2234,
    "definition": "A decision is a close choice when the second most likely play has at least half the probability of the first (internally: split).",
    "eligible": 180,
    "exclusions": {
     "grouped": 204,
     "invalid": 12,
     "not_audited": 527,
     "sparse": 1311
    },
    "groups": {
     "close": {
      "by_count": [
       0,
       0,
       0,
       0,
       0
      ],
      "differ": {
       "hitting": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       },
       "points": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       },
       "prime": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       },
       "race": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       }
      },
      "gap": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "loss1": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "loss2": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "n": 0,
      "p1": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "p2": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "psum": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "psum_bins": [
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0
      ]
     },
     "other": {
      "by_count": [
       54,
       81,
       37,
       8,
       0
      ],
      "differ": {
       "hitting": {
        "den": 180,
        "n": 74,
        "share": 0.4111111111111111,
        "small": true
       },
       "points": {
        "den": 180,
        "n": 29,
        "share": 0.16111111111111112,
        "small": true
       },
       "prime": {
        "den": 180,
        "n": 46,
        "share": 0.25555555555555554,
        "small": true
       },
       "race": {
        "den": 180,
        "n": 30,
        "share": 0.16666666666666666,
        "small": true
       }
      },
      "gap": {
       "mean": 0.11775344328747855,
       "median": 0.05112665891647339,
       "n": 180
      },
      "loss1": {
       "mean": 0.007187174922890133,
       "median": 0.0,
       "n": 180
      },
      "loss2": {
       "mean": 0.114272497759925,
       "median": 0.03965488076210022,
       "n": 180
      },
      "n": 180,
      "p1": {
       "mean": 0.9999988571238739,
       "median": 0.999999547881975,
       "n": 180
      },
      "p2": {
       "mean": 1.0218823356661373e-06,
       "median": 3.7885471755305115e-07,
       "n": 180
      },
      "psum": {
       "mean": 0.9999998790062097,
       "median": 1.0,
       "n": 180
      },
      "psum_bins": [
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       180
      ]
     }
    },
    "label": "Probabilities over plays, not sampled reasoning.",
    "margins": {
     "main": {
      "classes": {
       "none": {
        "close": {
         "den": 100,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 100,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       },
       "risk": {
        "close": {
         "den": 22,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 22,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       },
       "safety": {
        "close": {
         "den": 58,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 58,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       }
      },
      "margin": 0.001,
      "missed": {
       "close": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       },
       "group": 20,
       "na": "N/A: no close choices in this group",
       "other": {
        "den": 20,
        "n": 14,
        "share": 0.7,
        "small": true
       }
      },
      "stake": {
       "big": {
        "den": 9,
        "n": 0,
        "share": 0.0,
        "small": true
       },
       "medium": {
        "den": 9,
        "n": 0,
        "share": 0.0,
        "small": true
       },
       "small": {
        "den": 4,
        "n": 0,
        "share": 0.0,
        "small": true
       }
      }
     },
     "robust": {
      "classes": {
       "none": {
        "close": {
         "den": 100,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 100,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       },
       "risk": {
        "close": {
         "den": 18,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 18,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       },
       "safety": {
        "close": {
         "den": 62,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 62,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       }
      },
      "margin": 0.02,
      "missed": {
       "close": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       },
       "group": 17,
       "na": "N/A: no close choices in this group",
       "other": {
        "den": 17,
        "n": 11,
        "share": 0.6470588235294118,
        "small": true
       }
      },
      "stake": {
       "big": {
        "den": 9,
        "n": 0,
        "share": 0.0,
        "small": true
       },
       "medium": {
        "den": 9,
        "n": 0,
        "share": 0.0,
        "small": true
       },
       "small": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       }
      }
     }
    },
    "na": null,
    "risk_close": {
     "den": 0,
     "n": 0,
     "share": null,
     "small": false
    },
    "risk_close_of_eligible": {
     "den": 180,
     "n": 0,
     "share": 0.0,
     "small": true
    },
    "risk_pair_other": {
     "den": 180,
     "n": 73,
     "share": 0.40555555555555556,
     "small": true
    },
    "ties": {
     "close": 0,
     "member": 0,
     "order": 0,
     "positive": 0,
     "top_two": 0,
     "zero": 0
    }
   },
   "confidence": {
    "auroc_best": 0.49298512375082565,
    "auroc_blunder": 0.4941269108396723,
    "confident_badly_wrong": {
     "den": 1965,
     "n": 256,
     "share": 0.13027989821882952
    },
    "cutoffs": {
     "0.9": {
      "blunders": {
       "den": 1965,
       "n": 256,
       "share": 0.13027989821882952
      },
      "kept": {
       "den": 1965,
       "n": 1965,
       "share": 1.0
      },
      "mean_loss": 0.03294893177107697,
      "too_few": false
     },
     "0.99": {
      "blunders": {
       "den": 1965,
       "n": 256,
       "share": 0.13027989821882952
      },
      "kept": {
       "den": 1965,
       "n": 1965,
       "share": 1.0
      },
      "mean_loss": 0.03294893177107697,
      "too_few": false
     }
    },
    "ece": 0.3577601751433668,
    "held_out": {
     "blunders": null,
     "kept": null,
     "no_cutoff": 300,
     "runs": 300
    },
    "n": 1965,
    "surest": {
     "cut": 0.9999999737685851,
     "error_rate": 12.66115498009736,
     "n": 492,
     "too_few": false
    },
    "too_few": false
   },
   "quality": {
    "auroc": 0.49298512375082565,
    "best_share": 0.6183618361836184,
    "blunder_share": 0.13366336633663367,
    "brier": 0.3577602843168093,
    "clusters": 108,
    "confidence_n": 1965,
    "confident_badly_wrong": 0.13027989821882952,
    "cost_coverage": 1.0,
    "decisions": 2234,
    "ece": 0.3577601751433668,
    "engine_scored": 2234,
    "error_hi": 19.249331369928125,
    "error_lo": 15.09336266058623,
    "error_rate": 17.136322550683488,
    "first_date": "2026-10-24",
    "group_sizes": [
     14,
     15,
     16,
     17,
     18,
     19,
     20,
     21,
     22,
     23,
     24,
     25,
     26
    ],
    "grouped": 258,
    "invalid": 12,
    "last_date": "2026-10-26",
    "mean_confidence": 0.9999993608940029,
    "over_budget": 0.43384338433843384,
    "p95_loss": 0.19920729547738986,
    "probability_source": "logprobs",
    "retry_mean": 12.096897940913161,
    "scored": 2222,
    "time_median": 22.329500000000003,
    "time_p90": 107.5685,
    "tokens_in": 4214.513876454789,
    "tokens_out": 7172.9279319606085,
    "too_few": false,
    "usd_per_1000": 4.8688645487914055,
    "very_confident": 1.0,
    "worst_mean": 0.32988812928801187
   },
   "risk": {
    "main": {
     "by_call": {
      "missed": 9.928058675288296,
      "needless": 1.4753902983171892,
      "none": 0.8555873624920094,
      "right_risk": 1.6774957065749663,
      "right_safety": 1.6274058958305957,
      "wrong_risk": 1.5723846121804335
     },
     "calls": {
      "missed": 393.0,
      "needless": 51.0,
      "right_risk": 321.0,
      "right_safety": 872.0,
      "wrong_risk": 41.0
     },
     "confidence": {
      "auroc_right_risk": 0.5116875996810207,
      "missed_mean_p": 0.999998980094657,
      "missed_n": 337,
      "taken_grouped": 73,
      "taken_n": 340,
      "taken_other": 0
     },
     "error_rate": 17.136322550683488,
     "found": {
      "den": 755.0,
      "n": 321.0,
      "share": 0.4251655629139073
     },
     "good": {
      "den": 413.0,
      "n": 321.0,
      "share": 0.7772397094430993
     },
     "judgement": 68.4955479181764,
     "judgement_hi": 70.60623346328948,
     "judgement_lo": 66.40550901929036,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.0944700460829493,
      "x": 0.5748344370860927,
      "y": 0.0552546045503792
     },
     "margin": 0.001,
     "missed": {
      "den": 755.0,
      "n": 393.0,
      "share": 0.5205298013245033
     },
     "needless": {
      "den": 923.0,
      "n": 51.0,
      "share": 0.0552546045503792
     },
     "needless_by_loss": {
      "big": {
       "den": 923.0,
       "n": 26.0,
       "share": 0.028169014084507043
      },
      "medium": {
       "den": 923.0,
       "n": 19.0,
       "share": 0.020585048754062838
      },
      "small": {
       "den": 923.0,
       "n": 6.0,
       "share": 0.0065005417118093175
      }
     },
     "no_trade_off": 544.0,
     "reference": null,
     "right_safety": {
      "den": 923.0,
      "n": 872.0,
      "share": 0.9447453954496208
     },
     "risk_pays": 755.0,
     "safety_pays": 923.0,
     "scored": 2222,
     "sdt": {
      "aim": {
       "den": 362.0,
       "n": 321.0,
       "share": 0.8867403314917127
      },
      "beta": 1.3999731673375806,
      "caution": 0.8215136899585648,
      "caution_hi": 0.9081873467683751,
      "caution_lo": 0.7396362772005649,
      "cstar": 0.21844806532605013,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.540197070401678,
      "dprime_hi": 1.699281384645631,
      "dprime_lo": 1.393426194955194,
      "false_alarm": {
       "den": 923.0,
       "n": 51.0,
       "share": 0.0552546045503792
      },
      "false_alarm_rate": 0.055735930735930736,
      "hit": {
       "den": 755.0,
       "n": 362.0,
       "share": 0.4794701986754967
      },
      "hit_rate": 0.4794973544973545,
      "missed_loss": 0.1122653759617842,
      "missed_n": 393.0,
      "na": null,
      "needless_loss": 0.12856146050434486,
      "needless_n": 51.0
     },
     "size": {
      "bad": {
       "mean": 8.08695652173913,
       "n": 92.0,
       "too_few": true
      },
      "right": {
       "mean": 9.202492211838006,
       "n": 321.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 408.0,
        "n": 226.0,
        "share": 0.553921568627451
       },
       "missed": {
        "den": 408.0,
        "n": 165.0,
        "share": 0.40441176470588236
       },
       "risk_pays": 408.0,
       "wrong": {
        "den": 408.0,
        "n": 17.0,
        "share": 0.041666666666666664
       }
      },
      "medium": {
       "found": {
        "den": 246.0,
        "n": 81.0,
        "share": 0.32926829268292684
       },
       "missed": {
        "den": 246.0,
        "n": 148.0,
        "share": 0.6016260162601627
       },
       "risk_pays": 246.0,
       "wrong": {
        "den": 246.0,
        "n": 17.0,
        "share": 0.06910569105691057
       }
      },
      "small": {
       "found": {
        "den": 101.0,
        "n": 14.0,
        "share": 0.13861386138613863
       },
       "missed": {
        "den": 101.0,
        "n": 80.0,
        "share": 0.7920792079207921
       },
       "risk_pays": 101.0,
       "wrong": {
        "den": 101.0,
        "n": 7.0,
        "share": 0.06930693069306931
       }
      }
     },
     "trade_off": 1678.0,
     "wrong": {
      "den": 755.0,
      "n": 41.0,
      "share": 0.054304635761589407
     }
    },
    "robust": {
     "by_call": {
      "missed": 9.485449712358006,
      "needless": 1.5769740759414344,
      "none": 0.8555873624920094,
      "right_risk": 1.4174463618026993,
      "right_safety": 2.070014858760885,
      "wrong_risk": 1.730850179328455
     },
     "calls": {
      "missed": 313.0,
      "needless": 72.0,
      "right_risk": 293.0,
      "right_safety": 952.0,
      "wrong_risk": 48.0
     },
     "confidence": {
      "auroc_right_risk": 0.5196769840233337,
      "missed_mean_p": 0.9999988644438295,
      "missed_n": 274,
      "taken_grouped": 73,
      "taken_n": 340,
      "taken_other": 0
     },
     "error_rate": 17.136322550683488,
     "found": {
      "den": 654.0,
      "n": 293.0,
      "share": 0.44801223241590216
     },
     "good": {
      "den": 413.0,
      "n": 293.0,
      "share": 0.7094430992736077
     },
     "judgement": 68.8849866207951,
     "judgement_hi": 71.26410901909772,
     "judgement_lo": 66.49799540051556,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.1329639889196676,
      "x": 0.5519877675840978,
      "y": 0.0703125
     },
     "margin": 0.02,
     "missed": {
      "den": 654.0,
      "n": 313.0,
      "share": 0.4785932721712538
     },
     "needless": {
      "den": 1024.0,
      "n": 72.0,
      "share": 0.0703125
     },
     "needless_by_loss": {
      "big": {
       "den": 1024.0,
       "n": 28.0,
       "share": 0.02734375
      },
      "medium": {
       "den": 1024.0,
       "n": 24.0,
       "share": 0.0234375
      },
      "small": {
       "den": 1024.0,
       "n": 20.0,
       "share": 0.01953125
      }
     },
     "no_trade_off": 544.0,
     "reference": null,
     "right_safety": {
      "den": 1024.0,
      "n": 952.0,
      "share": 0.9296875
     },
     "risk_pays": 654.0,
     "safety_pays": 1024.0,
     "scored": 2222,
     "sdt": {
      "aim": {
       "den": 341.0,
       "n": 293.0,
       "share": 0.8592375366568915
      },
      "beta": 1.1316201641071533,
      "caution": 0.7083803736133817,
      "caution_hi": 0.791334112291597,
      "caution_lo": 0.6292878637762707,
      "cstar": 0.08113724900341372,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.52396563745126,
      "dprime_hi": 1.6979030935925108,
      "dprime_lo": 1.3584869024917763,
      "false_alarm": {
       "den": 1024.0,
       "n": 72.0,
       "share": 0.0703125
      },
      "false_alarm_rate": 0.07073170731707316,
      "hit": {
       "den": 654.0,
       "n": 341.0,
       "share": 0.5214067278287462
      },
      "hit_rate": 0.5213740458015267,
      "missed_loss": 0.13467520294478907,
      "missed_n": 313.0,
      "na": null,
      "needless_loss": 0.09733434435394075,
      "needless_n": 72.0
     },
     "size": {
      "bad": {
       "mean": 8.691666666666666,
       "n": 120.0,
       "too_few": true
      },
      "right": {
       "mean": 9.061433447098976,
       "n": 293.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 408.0,
        "n": 222.0,
        "share": 0.5441176470588235
       },
       "missed": {
        "den": 408.0,
        "n": 165.0,
        "share": 0.40441176470588236
       },
       "risk_pays": 408.0,
       "wrong": {
        "den": 408.0,
        "n": 21.0,
        "share": 0.051470588235294115
       }
      },
      "medium": {
       "found": {
        "den": 246.0,
        "n": 71.0,
        "share": 0.2886178861788618
       },
       "missed": {
        "den": 246.0,
        "n": 148.0,
        "share": 0.6016260162601627
       },
       "risk_pays": 246.0,
       "wrong": {
        "den": 246.0,
        "n": 27.0,
        "share": 0.10975609756097561
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 1678.0,
     "wrong": {
      "den": 654.0,
      "n": 48.0,
      "share": 0.07339449541284404
     }
    }
   },
   "slug": "deepseek-v4-1-flash-1017-on"
  },
  "DeepSeek-V4.1-Flash 1017 (off)": {
   "close": {
    "close": {
     "den": 537,
     "n": 62,
     "share": 0.1154562383612663,
     "small": false
    },
    "decisions": 2034,
    "definition": "A decision is a close choice when the second most likely play has at least half the probability of the first (internally: split).",
    "eligible": 537,
    "exclusions": {
     "grouped": 128,
     "not_audited": 428,
     "sparse": 941
    },
    "groups": {
     "close": {
      "by_count": [
       35,
       17,
       10,
       0,
       0
      ],
      "differ": {
       "hitting": {
        "den": 62,
        "n": 14,
        "share": 0.22580645161290322,
        "small": true
       },
       "points": {
        "den": 62,
        "n": 9,
        "share": 0.14516129032258066,
        "small": true
       },
       "prime": {
        "den": 62,
        "n": 11,
        "share": 0.1774193548387097,
        "small": true
       },
       "race": {
        "den": 62,
        "n": 3,
        "share": 0.04838709677419355,
        "small": true
       }
      },
      "gap": {
       "mean": 0.08187093273285896,
       "median": 0.01830267906188965,
       "n": 62
      },
      "loss1": {
       "mean": 0.058345549529598605,
       "median": 0.007862687110900879,
       "n": 62
      },
      "loss2": {
       "mean": 0.05141058564186096,
       "median": 0.0030472278594970703,
       "n": 62
      },
      "n": 62,
      "p1": {
       "mean": 0.552682160078469,
       "median": 0.5711854923091245,
       "n": 62
      },
      "p2": {
       "mean": 0.39150708795044326,
       "median": 0.39116546616694775,
       "n": 62
      },
      "psum": {
       "mean": 0.9441892480289122,
       "median": 0.9975346532051013,
       "n": 62
      },
      "psum_bins": [
       0,
       0,
       0,
       0,
       0,
       1,
       2,
       5,
       3,
       51
      ]
     },
     "other": {
      "by_count": [
       162,
       221,
       75,
       16,
       1
      ],
      "differ": {
       "hitting": {
        "den": 475,
        "n": 186,
        "share": 0.391578947368421,
        "small": false
       },
       "points": {
        "den": 475,
        "n": 66,
        "share": 0.13894736842105262,
        "small": false
       },
       "prime": {
        "den": 475,
        "n": 114,
        "share": 0.24,
        "small": false
       },
       "race": {
        "den": 475,
        "n": 57,
        "share": 0.12,
        "small": false
       }
      },
      "gap": {
       "mean": 0.12361543517363699,
       "median": 0.05491232872009277,
       "n": 475
      },
      "loss1": {
       "mean": 0.0264288414152045,
       "median": 0.0,
       "n": 475
      },
      "loss2": {
       "mean": 0.10664613221821032,
       "median": 0.03001570701599121,
       "n": 475
      },
      "n": 475,
      "p1": {
       "mean": 0.9411619819946151,
       "median": 0.9934835797050923,
       "n": 475
      },
      "p2": {
       "mean": 0.052264781278426445,
       "median": 0.005452273819588002,
       "n": 475
      },
      "psum": {
       "mean": 0.9934267632730415,
       "median": 0.9998145365020482,
       "n": 475
      },
      "psum_bins": [
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       1,
       6,
       468
      ]
     }
    },
    "label": "Probabilities over plays, not sampled reasoning.",
    "margins": {
     "main": {
      "classes": {
       "none": {
        "close": {
         "den": 291,
         "n": 36,
         "share": 0.12371134020618557,
         "small": false
        },
        "eligible": 291,
        "risk_close": {
         "den": 36,
         "n": 0,
         "share": 0.0,
         "small": true
        }
       },
       "risk": {
        "close": {
         "den": 72,
         "n": 14,
         "share": 0.19444444444444445,
         "small": true
        },
        "eligible": 72,
        "risk_close": {
         "den": 14,
         "n": 10,
         "share": 0.7142857142857143,
         "small": true
        }
       },
       "safety": {
        "close": {
         "den": 174,
         "n": 12,
         "share": 0.06896551724137931,
         "small": true
        },
        "eligible": 174,
        "risk_close": {
         "den": 12,
         "n": 3,
         "share": 0.25,
         "small": true
        }
       }
      },
      "margin": 0.001,
      "missed": {
       "adj_close": 0.7,
       "adj_other": 0.8625,
       "adjusted": -0.1625000000000001,
       "adjusted_hi": 0.16149350649350624,
       "adjusted_left_out": 3,
       "adjusted_lo": -0.3809641387419166,
       "close": {
        "den": 10,
        "n": 7,
        "share": 0.7,
        "small": true
       },
       "covered": {
        "den": 10,
        "n": 10,
        "share": 1.0,
        "small": true
       },
       "crude": -0.15416666666666667,
       "crude_hi": 0.1636054421768704,
       "crude_left_out": 3,
       "crude_lo": -0.3697886131196665,
       "group": 58,
       "matches": 15,
       "na": null,
       "other": {
        "den": 48,
        "n": 41,
        "share": 0.8541666666666666,
        "small": true
       },
       "resamples": 2000,
       "strata": 7,
       "strata_both": 3
      },
      "stake": {
       "big": {
        "den": 39,
        "n": 5,
        "share": 0.1282051282051282,
        "small": true
       },
       "medium": {
        "den": 18,
        "n": 6,
        "share": 0.3333333333333333,
        "small": true
       },
       "small": {
        "den": 15,
        "n": 3,
        "share": 0.2,
        "small": true
       }
      }
     },
     "robust": {
      "classes": {
       "none": {
        "close": {
         "den": 291,
         "n": 36,
         "share": 0.12371134020618557,
         "small": false
        },
        "eligible": 291,
        "risk_close": {
         "den": 36,
         "n": 0,
         "share": 0.0,
         "small": true
        }
       },
       "risk": {
        "close": {
         "den": 57,
         "n": 11,
         "share": 0.19298245614035087,
         "small": true
        },
        "eligible": 57,
        "risk_close": {
         "den": 11,
         "n": 7,
         "share": 0.6363636363636364,
         "small": true
        }
       },
       "safety": {
        "close": {
         "den": 189,
         "n": 15,
         "share": 0.07936507936507936,
         "small": true
        },
        "eligible": 189,
        "risk_close": {
         "den": 15,
         "n": 6,
         "share": 0.4,
         "small": true
        }
       }
      },
      "margin": 0.02,
      "missed": {
       "adj_close": 0.7142857142857143,
       "adj_other": 0.875,
       "adjusted": -0.1607142857142857,
       "adjusted_hi": 0.19434210526315793,
       "adjusted_left_out": 25,
       "adjusted_lo": -0.4482758620689655,
       "close": {
        "den": 7,
        "n": 5,
        "share": 0.7142857142857143,
        "small": true
       },
       "covered": {
        "den": 7,
        "n": 7,
        "share": 1.0,
        "small": true
       },
       "crude": -0.1278195488721804,
       "crude_hi": 0.23076923076923073,
       "crude_left_out": 25,
       "crude_lo": -0.41666666666666663,
       "group": 45,
       "matches": 15,
       "na": null,
       "other": {
        "den": 38,
        "n": 32,
        "share": 0.8421052631578947,
        "small": true
       },
       "resamples": 2000,
       "strata": 5,
       "strata_both": 3
      },
      "stake": {
       "big": {
        "den": 39,
        "n": 5,
        "share": 0.1282051282051282,
        "small": true
       },
       "medium": {
        "den": 18,
        "n": 6,
        "share": 0.3333333333333333,
        "small": true
       },
       "small": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       }
      }
     }
    },
    "na": null,
    "risk_close": {
     "den": 62,
     "n": 13,
     "share": 0.20967741935483872,
     "small": true
    },
    "risk_close_of_eligible": {
     "den": 537,
     "n": 13,
     "share": 0.024208566108007448,
     "small": false
    },
    "risk_pair_other": {
     "den": 475,
     "n": 184,
     "share": 0.3873684210526316,
     "small": false
    },
    "ties": {
     "close": 0,
     "member": 0,
     "order": 0,
     "positive": 0,
     "top_two": 0,
     "zero": 0
    }
   },
   "confidence": {
    "auroc_best": 0.7225684595086421,
    "auroc_blunder": 0.6631993464052287,
    "confident_badly_wrong": {
     "den": 1865,
     "n": 158,
     "share": 0.08471849865951743
    },
    "cutoffs": {
     "0.9": {
      "blunders": {
       "den": 1030,
       "n": 158,
       "share": 0.1533980582524272
      },
      "kept": {
       "den": 1865,
       "n": 1030,
       "share": 0.5522788203753352
      },
      "mean_loss": 0.042544244909749444,
      "too_few": false
     },
     "0.99": {
      "blunders": {
       "den": 597,
       "n": 57,
       "share": 0.09547738693467336
      },
      "kept": {
       "den": 1865,
       "n": 597,
       "share": 0.3201072386058981
      },
      "mean_loss": 0.024426785346051957,
      "too_few": false
     }
    },
    "ece": 0.3531301983718327,
    "held_out": {
     "blunders": 0.0322041106805878,
     "kept": 0.024739976801609557,
     "no_cutoff": 0,
     "runs": 300
    },
    "n": 1865,
    "surest": {
     "cut": 0.9960446726243035,
     "error_rate": 11.455256563693094,
     "n": 467,
     "too_few": false
    },
    "too_few": false
   },
   "quality": {
    "auroc": 0.7225684595086421,
    "best_share": 0.4773844641101278,
    "blunder_share": 0.2359882005899705,
    "brier": 0.34631154986801277,
    "clusters": 102,
    "confidence_n": 1865,
    "confident_badly_wrong": 0.08471849865951743,
    "cost_coverage": 1.0,
    "decisions": 2034,
    "ece": 0.3531301983718327,
    "engine_scored": 2034,
    "error_hi": 38.81113833697785,
    "error_lo": 31.811222711720227,
    "error_rate": 35.248935273907996,
    "first_date": "2026-10-24",
    "group_sizes": [
     14,
     15,
     16,
     17,
     18,
     19,
     20,
     21,
     22,
     23,
     24,
     25,
     26
    ],
    "grouped": 169,
    "invalid": 0,
    "last_date": "2026-10-26",
    "mean_confidence": 0.8453553994442188,
    "over_budget": 0.0,
    "p95_loss": 0.36535311341285687,
    "probability_source": "logprobs",
    "retry_mean": 24.175966076696167,
    "scored": 2034,
    "time_median": 0.8434999999999999,
    "time_p90": 1.3931,
    "tokens_in": 3845.5039331366766,
    "tokens_out": 1.233038348082596,
    "too_few": false,
    "usd_per_1000": 0.24391185250737463,
    "very_confident": 0.5522788203753352,
    "worst_mean": 0.5914421931785696
   },
   "risk": {
    "main": {
     "by_call": {
      "missed": 21.425522666061873,
      "needless": 2.493309499943151,
      "none": 3.4666398545633728,
      "right_risk": 1.613494430026708,
      "right_safety": 3.7507157161055296,
      "wrong_risk": 2.499253107207363
     },
     "calls": {
      "missed": 487.0,
      "needless": 62.0,
      "right_risk": 181.0,
      "right_safety": 683.0,
      "wrong_risk": 45.0
     },
     "confidence": {
      "auroc_right_risk": 0.5830079940509388,
      "missed_mean_p": 0.8097810685073139,
      "missed_n": 437,
      "taken_grouped": 26,
      "taken_n": 262,
      "taken_other": 0
     },
     "error_rate": 35.248935273907996,
     "found": {
      "den": 713.0,
      "n": 181.0,
      "share": 0.2538569424964937
     },
     "good": {
      "den": 288.0,
      "n": 181.0,
      "share": 0.6284722222222222
     },
     "judgement": 58.53177329932133,
     "judgement_hi": 60.48651420777238,
     "judgement_lo": 56.57573275267902,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.08458646616541353,
      "x": 0.7461430575035063,
      "y": 0.08322147651006712
     },
     "margin": 0.001,
     "missed": {
      "den": 713.0,
      "n": 487.0,
      "share": 0.6830294530154277
     },
     "needless": {
      "den": 745.0,
      "n": 62.0,
      "share": 0.08322147651006712
     },
     "needless_by_loss": {
      "big": {
       "den": 745.0,
       "n": 38.0,
       "share": 0.05100671140939597
      },
      "medium": {
       "den": 745.0,
       "n": 19.0,
       "share": 0.025503355704697986
      },
      "small": {
       "den": 745.0,
       "n": 5.0,
       "share": 0.006711409395973154
      }
     },
     "no_trade_off": 576.0,
     "reference": null,
     "right_safety": {
      "den": 745.0,
      "n": 683.0,
      "share": 0.9167785234899329
     },
     "risk_pays": 713.0,
     "safety_pays": 745.0,
     "scored": 2034,
     "sdt": {
      "aim": {
       "den": 226.0,
       "n": 181.0,
       "share": 0.8008849557522124
      },
      "beta": 0.9550998596341542,
      "caution": 0.9277765071014472,
      "caution_hi": 1.0168866985170726,
      "caution_lo": 0.8449330118754742,
      "cstar": -0.05078318117577723,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 0.9046179826151297,
      "dprime_hi": 1.0664848036377832,
      "dprime_lo": 0.7448523660363385,
      "false_alarm": {
       "den": 745.0,
       "n": 62.0,
       "share": 0.08322147651006712
      },
      "false_alarm_rate": 0.08378016085790885,
      "hit": {
       "den": 713.0,
       "n": 226.0,
       "share": 0.3169705469845722
      },
      "hit_rate": 0.3172268907563025,
      "missed_loss": 0.17897130637687825,
      "missed_n": 487.0,
      "na": null,
      "needless_loss": 0.16359327493175382,
      "needless_n": 62.0
     },
     "size": {
      "bad": {
       "mean": 8.607476635514018,
       "n": 107.0,
       "too_few": true
      },
      "right": {
       "mean": 6.386740331491713,
       "n": 181.0,
       "too_few": true
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 415.0,
        "n": 138.0,
        "share": 0.3325301204819277
       },
       "missed": {
        "den": 415.0,
        "n": 257.0,
        "share": 0.619277108433735
       },
       "risk_pays": 415.0,
       "wrong": {
        "den": 415.0,
        "n": 20.0,
        "share": 0.04819277108433735
       }
      },
      "medium": {
       "found": {
        "den": 220.0,
        "n": 37.0,
        "share": 0.16818181818181818
       },
       "missed": {
        "den": 220.0,
        "n": 163.0,
        "share": 0.740909090909091
       },
       "risk_pays": 220.0,
       "wrong": {
        "den": 220.0,
        "n": 20.0,
        "share": 0.09090909090909091
       }
      },
      "small": {
       "found": {
        "den": 78.0,
        "n": 6.0,
        "share": 0.07692307692307693
       },
       "missed": {
        "den": 78.0,
        "n": 67.0,
        "share": 0.8589743589743589
       },
       "risk_pays": 78.0,
       "wrong": {
        "den": 78.0,
        "n": 5.0,
        "share": 0.0641025641025641
       }
      }
     },
     "trade_off": 1458.0,
     "wrong": {
      "den": 713.0,
      "n": 45.0,
      "share": 0.06311360448807854
     }
    },
    "robust": {
     "by_call": {
      "missed": 21.03093432788183,
      "needless": 2.5999038649636854,
      "none": 3.4666398545633728,
      "right_risk": 1.4210854059006495,
      "right_safety": 4.145304054285573,
      "wrong_risk": 2.585067766312887
     },
     "calls": {
      "missed": 420.0,
      "needless": 73.0,
      "right_risk": 168.0,
      "right_safety": 750.0,
      "wrong_risk": 47.0
     },
     "confidence": {
      "auroc_right_risk": 0.597109791929004,
      "missed_mean_p": 0.8011296431559521,
      "missed_n": 377,
      "taken_grouped": 26,
      "taken_n": 262,
      "taken_other": 0
     },
     "error_rate": 35.248935273907996,
     "found": {
      "den": 635.0,
      "n": 168.0,
      "share": 0.26456692913385826
     },
     "good": {
      "den": 288.0,
      "n": 168.0,
      "share": 0.5833333333333334
     },
     "judgement": 58.79335253202706,
     "judgement_hi": 60.92868583217564,
     "judgement_lo": 56.71684645630735,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.1006423982869379,
      "x": 0.7354330708661417,
      "y": 0.08869987849331713
     },
     "margin": 0.02,
     "missed": {
      "den": 635.0,
      "n": 420.0,
      "share": 0.6614173228346457
     },
     "needless": {
      "den": 823.0,
      "n": 73.0,
      "share": 0.08869987849331713
     },
     "needless_by_loss": {
      "big": {
       "den": 823.0,
       "n": 41.0,
       "share": 0.04981773997569866
      },
      "medium": {
       "den": 823.0,
       "n": 20.0,
       "share": 0.024301336573511544
      },
      "small": {
       "den": 823.0,
       "n": 12.0,
       "share": 0.014580801944106925
      }
     },
     "no_trade_off": 576.0,
     "reference": null,
     "right_safety": {
      "den": 823.0,
      "n": 750.0,
      "share": 0.9113001215066828
     },
     "risk_pays": 635.0,
     "safety_pays": 823.0,
     "scored": 2034,
     "sdt": {
      "aim": {
       "den": 215.0,
       "n": 168.0,
       "share": 0.7813953488372093
      },
      "beta": 0.9218309019054789,
      "caution": 0.8806722422969384,
      "caution_hi": 0.9747313180641741,
      "caution_lo": 0.794206808701104,
      "cstar": -0.08751392600169884,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 0.9300631286692416,
      "dprime_hi": 1.101415830472527,
      "dprime_lo": 0.7669174782389637,
      "false_alarm": {
       "den": 823.0,
       "n": 73.0,
       "share": 0.08869987849331713
      },
      "false_alarm_rate": 0.0891990291262136,
      "hit": {
       "den": 635.0,
       "n": 215.0,
       "share": 0.33858267716535434
      },
      "hit_rate": 0.3388364779874214,
      "missed_loss": 0.203699621061484,
      "missed_n": 420.0,
      "na": null,
      "needless_loss": 0.1448823140092092,
      "needless_n": 73.0
     },
     "size": {
      "bad": {
       "mean": 8.35,
       "n": 120.0,
       "too_few": true
      },
      "right": {
       "mean": 6.398809523809524,
       "n": 168.0,
       "too_few": true
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 415.0,
        "n": 135.0,
        "share": 0.3253012048192771
       },
       "missed": {
        "den": 415.0,
        "n": 257.0,
        "share": 0.619277108433735
       },
       "risk_pays": 415.0,
       "wrong": {
        "den": 415.0,
        "n": 23.0,
        "share": 0.05542168674698795
       }
      },
      "medium": {
       "found": {
        "den": 220.0,
        "n": 33.0,
        "share": 0.15
       },
       "missed": {
        "den": 220.0,
        "n": 163.0,
        "share": 0.740909090909091
       },
       "risk_pays": 220.0,
       "wrong": {
        "den": 220.0,
        "n": 24.0,
        "share": 0.10909090909090909
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 1458.0,
     "wrong": {
      "den": 635.0,
      "n": 47.0,
      "share": 0.07401574803149606
     }
    }
   },
   "slug": "deepseek-v4-1-flash-1017-off"
  },
  "Fastino GLiDE": {
   "close": {
    "close": {
     "den": 1947,
     "n": 1015,
     "share": 0.5213148433487417,
     "small": false
    },
    "decisions": 2159,
    "definition": "A decision is a close choice when the second most likely play has at least half the probability of the first (internally: split).",
    "eligible": 1947,
    "exclusions": {
     "grouped": 2,
     "not_audited": 210
    },
    "groups": {
     "close": {
      "by_count": [
       278,
       355,
       260,
       108,
       14
      ],
      "differ": {
       "hitting": {
        "den": 1015,
        "n": 516,
        "share": 0.5083743842364532,
        "small": false
       },
       "points": {
        "den": 1015,
        "n": 337,
        "share": 0.33201970443349754,
        "small": false
       },
       "prime": {
        "den": 1015,
        "n": 258,
        "share": 0.2541871921182266,
        "small": false
       },
       "race": {
        "den": 1015,
        "n": 144,
        "share": 0.14187192118226602,
        "small": false
       }
      },
      "gap": {
       "mean": 0.13082730315001728,
       "median": 0.07409167289733887,
       "n": 1015
      },
      "loss1": {
       "mean": 0.08362524483004227,
       "median": 0.020867586135864258,
       "n": 1015
      },
      "loss2": {
       "mean": 0.11809450951703077,
       "median": 0.05040782690048218,
       "n": 1015
      },
      "n": 1015,
      "p1": {
       "mean": 0.3171896133046298,
       "median": 0.2953636391736017,
       "n": 1015
      },
      "p2": {
       "mean": 0.2283769408817031,
       "median": 0.21476192290534157,
       "n": 1015
      },
      "psum": {
       "mean": 0.5455665541863328,
       "median": 0.51247696790651,
       "n": 1015
      },
      "psum_bins": [
       3,
       37,
       95,
       169,
       183,
       154,
       114,
       93,
       72,
       95
      ]
     },
     "other": {
      "by_count": [
       181,
       386,
       257,
       99,
       9
      ],
      "differ": {
       "hitting": {
        "den": 932,
        "n": 514,
        "share": 0.5515021459227468,
        "small": false
       },
       "points": {
        "den": 932,
        "n": 338,
        "share": 0.3626609442060086,
        "small": false
       },
       "prime": {
        "den": 932,
        "n": 285,
        "share": 0.30579399141630903,
        "small": false
       },
       "race": {
        "den": 932,
        "n": 96,
        "share": 0.10300429184549356,
        "small": false
       }
      },
      "gap": {
       "mean": 0.14277861660642174,
       "median": 0.08639171719551086,
       "n": 932
      },
      "loss1": {
       "mean": 0.04885719169746653,
       "median": 0.0,
       "n": 932
      },
      "loss2": {
       "mean": 0.13544390727905756,
       "median": 0.07896280288696289,
       "n": 932
      },
      "n": 932,
      "p1": {
       "mean": 0.5868863868085308,
       "median": 0.5850139064341167,
       "n": 932
      },
      "p2": {
       "mean": 0.15113918529417245,
       "median": 0.14623903295475432,
       "n": 932
      },
      "psum": {
       "mean": 0.7380255721027031,
       "median": 0.7623513760588149,
       "n": 932
      },
      "psum_bins": [
       0,
       2,
       11,
       39,
       68,
       124,
       133,
       146,
       182,
       227
      ]
     }
    },
    "label": "Probabilities over plays, not sampled reasoning.",
    "margins": {
     "main": {
      "classes": {
       "none": {
        "close": {
         "den": 521,
         "n": 250,
         "share": 0.4798464491362764,
         "small": false
        },
        "eligible": 521,
        "risk_close": {
         "den": 250,
         "n": 0,
         "share": 0.0,
         "small": false
        }
       },
       "risk": {
        "close": {
         "den": 623,
         "n": 358,
         "share": 0.5746388443017657,
         "small": false
        },
        "eligible": 623,
        "risk_close": {
         "den": 358,
         "n": 213,
         "share": 0.5949720670391061,
         "small": false
        }
       },
       "safety": {
        "close": {
         "den": 803,
         "n": 407,
         "share": 0.5068493150684932,
         "small": false
        },
        "eligible": 803,
        "risk_close": {
         "den": 407,
         "n": 204,
         "share": 0.5012285012285013,
         "small": false
        }
       }
      },
      "margin": 0.001,
      "missed": {
       "adj_close": 0.5346534653465347,
       "adj_other": 0.7813521688303283,
       "adjusted": -0.24669870348379364,
       "adjusted_hi": -0.13228929100982256,
       "adjusted_left_out": 0,
       "adjusted_lo": -0.3457253553635412,
       "close": {
        "den": 213,
        "n": 115,
        "share": 0.539906103286385,
        "small": false
       },
       "covered": {
        "den": 213,
        "n": 202,
        "share": 0.9483568075117371,
        "small": false
       },
       "crude": -0.1904309753653004,
       "crude_hi": -0.11483846472707096,
       "crude_left_out": 0,
       "crude_lo": -0.26165056717476076,
       "group": 391,
       "matches": 18,
       "na": null,
       "other": {
        "den": 178,
        "n": 130,
        "share": 0.7303370786516854,
        "small": true
       },
       "resamples": 2000,
       "strata": 35,
       "strata_both": 26
      },
      "stake": {
       "big": {
        "den": 335,
        "n": 196,
        "share": 0.5850746268656717,
        "small": false
       },
       "medium": {
        "den": 213,
        "n": 117,
        "share": 0.5492957746478874,
        "small": false
       },
       "small": {
        "den": 75,
        "n": 45,
        "share": 0.6,
        "small": true
       }
      }
     },
     "robust": {
      "classes": {
       "none": {
        "close": {
         "den": 521,
         "n": 250,
         "share": 0.4798464491362764,
         "small": false
        },
        "eligible": 521,
        "risk_close": {
         "den": 250,
         "n": 0,
         "share": 0.0,
         "small": false
        }
       },
       "risk": {
        "close": {
         "den": 548,
         "n": 313,
         "share": 0.5711678832116789,
         "small": false
        },
        "eligible": 548,
        "risk_close": {
         "den": 313,
         "n": 188,
         "share": 0.6006389776357828,
         "small": false
        }
       },
       "safety": {
        "close": {
         "den": 878,
         "n": 452,
         "share": 0.5148063781321185,
         "small": false
        },
        "eligible": 878,
        "risk_close": {
         "den": 452,
         "n": 229,
         "share": 0.5066371681415929,
         "small": false
        }
       }
      },
      "margin": 0.02,
      "missed": {
       "adj_close": 0.5,
       "adj_other": 0.7850176297962154,
       "adjusted": -0.28501762979621537,
       "adjusted_hi": -0.1288663160417297,
       "adjusted_left_out": 0,
       "adjusted_lo": -0.4060726674562559,
       "close": {
        "den": 188,
        "n": 96,
        "share": 0.5106382978723404,
        "small": true
       },
       "covered": {
        "den": 188,
        "n": 178,
        "share": 0.9468085106382979,
        "small": true
       },
       "crude": -0.2296214423873999,
       "crude_hi": -0.13214734637780975,
       "crude_left_out": 0,
       "crude_lo": -0.3228111463452042,
       "group": 342,
       "matches": 18,
       "na": null,
       "other": {
        "den": 154,
        "n": 114,
        "share": 0.7402597402597403,
        "small": true
       },
       "resamples": 2000,
       "strata": 35,
       "strata_both": 25
      },
      "stake": {
       "big": {
        "den": 335,
        "n": 196,
        "share": 0.5850746268656717,
        "small": false
       },
       "medium": {
        "den": 213,
        "n": 117,
        "share": 0.5492957746478874,
        "small": false
       },
       "small": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       }
      }
     }
    },
    "na": null,
    "risk_close": {
     "den": 1015,
     "n": 417,
     "share": 0.41083743842364534,
     "small": false
    },
    "risk_close_of_eligible": {
     "den": 1947,
     "n": 417,
     "share": 0.21417565485362094,
     "small": false
    },
    "risk_pair_other": {
     "den": 932,
     "n": 456,
     "share": 0.4892703862660944,
     "small": false
    },
    "ties": {
     "close": 0,
     "member": 0,
     "order": 0,
     "positive": 0,
     "top_two": 0,
     "zero": 0
    }
   },
   "confidence": {
    "auroc_best": 0.7174853240582184,
    "auroc_blunder": 0.6847663491958794,
    "confident_badly_wrong": {
     "den": 2157,
     "n": 0,
     "share": 0.0
    },
    "cutoffs": {
     "0.9": {
      "blunders": {
       "den": 30,
       "n": 0,
       "share": 0.0
      },
      "kept": {
       "den": 2157,
       "n": 30,
       "share": 0.013908205841446454
      },
      "mean_loss": 0.0019647101561228435,
      "too_few": true
     },
     "0.99": {
      "blunders": {
       "den": 0,
       "n": 0,
       "share": null
      },
      "kept": {
       "den": 2157,
       "n": 0,
       "share": 0.0
      },
      "mean_loss": null,
      "too_few": true
     }
    },
    "ece": 0.04183461456260687,
    "held_out": {
     "blunders": 0.012109671051079095,
     "kept": 0.039876170830604844,
     "no_cutoff": 0,
     "runs": 300
    },
    "n": 2157,
    "surest": {
     "cut": 0.5915431584176731,
     "error_rate": 10.747340100782889,
     "n": 540,
     "too_few": false
    },
    "too_few": false
   },
   "quality": {
    "auroc": 0.7174853240582184,
    "best_share": 0.48448355720240854,
    "blunder_share": 0.24085224641037517,
    "brier": 0.21477183993641863,
    "clusters": 104,
    "confidence_n": 2157,
    "confident_badly_wrong": 0.0,
    "cost_coverage": 1.0,
    "decisions": 2159,
    "ece": 0.04183461456260687,
    "engine_scored": 2159,
    "error_hi": 36.40387392537881,
    "error_lo": 30.166719096109453,
    "error_rate": 33.1826399685664,
    "first_date": "2026-10-20",
    "group_sizes": [
     132,
     152
    ],
    "grouped": 2,
    "invalid": 0,
    "last_date": "2026-10-25",
    "mean_confidence": 0.44533979752091285,
    "over_budget": 0.0,
    "own_cost": {
     "decisions": 2159,
     "input_per_mtok": 0.04,
     "kind": "tokens",
     "output_per_mtok": 0,
     "source": "https://docs.fastino.ai/pricing",
     "tokens_in": 8267523,
     "tokens_out": 0,
     "usd": 0.33070092,
     "usd_per_1000": 0.15317319129226495
    },
    "p95_loss": 0.32329131364822383,
    "probability_source": "decision",
    "retry_mean": 0.0,
    "scored": 2159,
    "time_median": 0.479,
    "time_p90": 0.792,
    "tokens_in": 3829.3297823066237,
    "tokens_out": 0.0,
    "too_few": false,
    "usd_per_1000": 0.15317319129226495,
    "very_confident": 0.013908205841446454,
    "worst_mean": 0.5151958253096651
   },
   "risk": {
    "main": {
     "by_call": {
      "missed": 10.97331048714557,
      "needless": 7.182986496664297,
      "none": 3.2413427395752152,
      "right_risk": 2.0277315458250906,
      "right_safety": 4.966338464042113,
      "wrong_risk": 4.790930235314115
     },
     "calls": {
      "missed": 344.0,
      "needless": 144.0,
      "right_risk": 249.0,
      "right_safety": 756.0,
      "wrong_risk": 82.0
     },
     "confidence": {
      "auroc_right_risk": 0.57808224046629,
      "missed_mean_p": 0.39748961711809394,
      "missed_n": 343,
      "taken_grouped": 0,
      "taken_n": 475,
      "taken_other": 0
     },
     "error_rate": 33.1826399685664,
     "found": {
      "den": 675.0,
      "n": 249.0,
      "share": 0.3688888888888889
     },
     "good": {
      "den": 475.0,
      "n": 249.0,
      "share": 0.5242105263157895
     },
     "judgement": 60.44444444444444,
     "judgement_hi": 62.56019595841687,
     "judgement_lo": 58.2510296368265,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.19248826291079812,
      "x": 0.6311111111111111,
      "y": 0.16
     },
     "margin": 0.001,
     "missed": {
      "den": 675.0,
      "n": 344.0,
      "share": 0.5096296296296297
     },
     "needless": {
      "den": 900.0,
      "n": 144.0,
      "share": 0.16
     },
     "needless_by_loss": {
      "big": {
       "den": 900.0,
       "n": 107.0,
       "share": 0.11888888888888889
      },
      "medium": {
       "den": 900.0,
       "n": 27.0,
       "share": 0.03
      },
      "small": {
       "den": 900.0,
       "n": 10.0,
       "share": 0.011111111111111112
      }
     },
     "no_trade_off": 584.0,
     "reference": null,
     "right_safety": {
      "den": 900.0,
      "n": 756.0,
      "share": 0.84
     },
     "risk_pays": 675.0,
     "safety_pays": 900.0,
     "scored": 2159,
     "sdt": {
      "aim": {
       "den": 331.0,
       "n": 249.0,
       "share": 0.7522658610271903
      },
      "beta": 2.0849808452391456,
      "caution": 0.5085063402105007,
      "caution_hi": 0.5843475494646305,
      "caution_lo": 0.4335263315971032,
      "cstar": 0.758419612849244,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 0.9688036224597952,
      "dprime_hi": 1.0851968189006485,
      "dprime_lo": 0.8528400529413329,
      "false_alarm": {
       "den": 900.0,
       "n": 144.0,
       "share": 0.16
      },
      "false_alarm_rate": 0.16037735849056603,
      "hit": {
       "den": 675.0,
       "n": 331.0,
       "share": 0.49037037037037035
      },
      "hit_rate": 0.49038461538461536,
      "missed_loss": 0.1377405659403912,
      "missed_n": 344.0,
      "na": null,
      "needless_loss": 0.21538983119858635,
      "needless_n": 144.0
     },
     "size": {
      "bad": {
       "mean": 9.938053097345133,
       "n": 226.0,
       "too_few": false
      },
      "right": {
       "mean": 12.160642570281125,
       "n": 249.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 370.0,
        "n": 169.0,
        "share": 0.45675675675675675
       },
       "missed": {
        "den": 370.0,
        "n": 157.0,
        "share": 0.4243243243243243
       },
       "risk_pays": 370.0,
       "wrong": {
        "den": 370.0,
        "n": 44.0,
        "share": 0.11891891891891893
       }
      },
      "medium": {
       "found": {
        "den": 223.0,
        "n": 66.0,
        "share": 0.29596412556053814
       },
       "missed": {
        "den": 223.0,
        "n": 133.0,
        "share": 0.5964125560538116
       },
       "risk_pays": 223.0,
       "wrong": {
        "den": 223.0,
        "n": 24.0,
        "share": 0.10762331838565023
       }
      },
      "small": {
       "found": {
        "den": 82.0,
        "n": 14.0,
        "share": 0.17073170731707318
       },
       "missed": {
        "den": 82.0,
        "n": 54.0,
        "share": 0.6585365853658537
       },
       "risk_pays": 82.0,
       "wrong": {
        "den": 82.0,
        "n": 14.0,
        "share": 0.17073170731707318
       }
      }
     },
     "trade_off": 1575.0,
     "wrong": {
      "den": 675.0,
      "n": 82.0,
      "share": 0.12148148148148148
     }
    },
    "robust": {
     "by_call": {
      "missed": 10.52082161237488,
      "needless": 7.564660808784976,
      "none": 3.2413427395752152,
      "right_risk": 1.913418574927305,
      "right_safety": 5.418827338812803,
      "wrong_risk": 4.523568894091222
     },
     "calls": {
      "missed": 290.0,
      "needless": 172.0,
      "right_risk": 228.0,
      "right_safety": 810.0,
      "wrong_risk": 75.0
     },
     "confidence": {
      "auroc_right_risk": 0.5655053625967753,
      "missed_mean_p": 0.40052845836284556,
      "missed_n": 289,
      "taken_grouped": 0,
      "taken_n": 475,
      "taken_other": 0
     },
     "error_rate": 33.1826399685664,
     "found": {
      "den": 593.0,
      "n": 228.0,
      "share": 0.3844856661045531
     },
     "good": {
      "den": 475.0,
      "n": 228.0,
      "share": 0.48
     },
     "judgement": 60.466645830685906,
     "judgement_hi": 62.63847841930668,
     "judgement_lo": 58.205254770936904,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.2054794520547945,
      "x": 0.6155143338954469,
      "y": 0.17515274949083504
     },
     "margin": 0.02,
     "missed": {
      "den": 593.0,
      "n": 290.0,
      "share": 0.48903878583473864
     },
     "needless": {
      "den": 982.0,
      "n": 172.0,
      "share": 0.17515274949083504
     },
     "needless_by_loss": {
      "big": {
       "den": 982.0,
       "n": 113.0,
       "share": 0.11507128309572301
      },
      "medium": {
       "den": 982.0,
       "n": 34.0,
       "share": 0.034623217922606926
      },
      "small": {
       "den": 982.0,
       "n": 25.0,
       "share": 0.025458248472505093
      }
     },
     "no_trade_off": 584.0,
     "reference": null,
     "right_safety": {
      "den": 982.0,
      "n": 810.0,
      "share": 0.824847250509165
     },
     "risk_pays": 593.0,
     "safety_pays": 982.0,
     "scored": 2159,
     "sdt": {
      "aim": {
       "den": 303.0,
       "n": 228.0,
       "share": 0.7524752475247525
      },
      "beta": 2.0075489709817567,
      "caution": 0.4526417489540398,
      "caution_hi": 0.5301378237437027,
      "caution_lo": 0.38137316358783147,
      "cstar": 0.725839824294178,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 0.9601492467463787,
      "dprime_hi": 1.0699870053717886,
      "dprime_lo": 0.8571238258597575,
      "false_alarm": {
       "den": 982.0,
       "n": 172.0,
       "share": 0.17515274949083504
      },
      "false_alarm_rate": 0.17548321464903358,
      "hit": {
       "den": 593.0,
       "n": 303.0,
       "share": 0.5109612141652614
      },
      "hit_rate": 0.5109427609427609,
      "missed_loss": 0.15665140593874044,
      "missed_n": 290.0,
      "na": null,
      "needless_loss": 0.18990817076938096,
      "needless_n": 172.0
     },
     "size": {
      "bad": {
       "mean": 10.097165991902834,
       "n": 247.0,
       "too_few": false
      },
      "right": {
       "mean": 12.192982456140351,
       "n": 228.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 370.0,
        "n": 166.0,
        "share": 0.4486486486486487
       },
       "missed": {
        "den": 370.0,
        "n": 157.0,
        "share": 0.4243243243243243
       },
       "risk_pays": 370.0,
       "wrong": {
        "den": 370.0,
        "n": 47.0,
        "share": 0.12702702702702703
       }
      },
      "medium": {
       "found": {
        "den": 223.0,
        "n": 62.0,
        "share": 0.27802690582959644
       },
       "missed": {
        "den": 223.0,
        "n": 133.0,
        "share": 0.5964125560538116
       },
       "risk_pays": 223.0,
       "wrong": {
        "den": 223.0,
        "n": 28.0,
        "share": 0.12556053811659193
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 1575.0,
     "wrong": {
      "den": 593.0,
      "n": 75.0,
      "share": 0.12647554806070826
     }
    }
   },
   "slug": "fastino-glide"
  },
  "GLM-5.3 (low)": {
   "close": {
    "close": {
     "den": 527,
     "n": 29,
     "share": 0.05502846299810247,
     "small": false
    },
    "decisions": 1989,
    "definition": "A decision is a close choice when the second most likely play has at least half the probability of the first (internally: split).",
    "eligible": 527,
    "exclusions": {
     "grouped": 184,
     "no_probabilities": 1,
     "not_audited": 187,
     "sparse": 1090
    },
    "groups": {
     "close": {
      "by_count": [
       17,
       7,
       2,
       3,
       0
      ],
      "differ": {
       "hitting": {
        "den": 29,
        "n": 9,
        "share": 0.3103448275862069,
        "small": true
       },
       "points": {
        "den": 29,
        "n": 5,
        "share": 0.1724137931034483,
        "small": true
       },
       "prime": {
        "den": 29,
        "n": 5,
        "share": 0.1724137931034483,
        "small": true
       },
       "race": {
        "den": 29,
        "n": 1,
        "share": 0.034482758620689655,
        "small": true
       }
      },
      "gap": {
       "mean": 0.11277328482989607,
       "median": 0.041048407554626465,
       "n": 29
      },
      "loss1": {
       "mean": 0.07840964506412375,
       "median": 0.007918238639831543,
       "n": 29
      },
      "loss2": {
       "mean": 0.0369816648549047,
       "median": 0.0,
       "n": 29
      },
      "n": 29,
      "p1": {
       "mean": 0.5283065239599,
       "median": 0.5247439798607527,
       "n": 29
      },
      "p2": {
       "mean": 0.38207554177102593,
       "median": 0.3924361222727896,
       "n": 29
      },
      "psum": {
       "mean": 0.910382065730926,
       "median": 0.9634273109657572,
       "n": 29
      },
      "psum_bins": [
       0,
       0,
       0,
       0,
       0,
       0,
       3,
       4,
       2,
       20
      ]
     },
     "other": {
      "by_count": [
       160,
       223,
       100,
       15,
       0
      ],
      "differ": {
       "hitting": {
        "den": 498,
        "n": 202,
        "share": 0.40562248995983935,
        "small": false
       },
       "points": {
        "den": 498,
        "n": 72,
        "share": 0.14457831325301204,
        "small": false
       },
       "prime": {
        "den": 498,
        "n": 128,
        "share": 0.2570281124497992,
        "small": false
       },
       "race": {
        "den": 498,
        "n": 66,
        "share": 0.13253012048192772,
        "small": false
       }
      },
      "gap": {
       "mean": 0.1340060979127884,
       "median": 0.05392920970916748,
       "n": 498
      },
      "loss1": {
       "mean": 0.025749141552362097,
       "median": 0.0,
       "n": 498
      },
      "loss2": {
       "mean": 0.12745362322732626,
       "median": 0.04200613498687744,
       "n": 498
      },
      "n": 498,
      "p1": {
       "mean": 0.954167475419293,
       "median": 0.9824774411111221,
       "n": 498
      },
      "p2": {
       "mean": 0.03564311215785503,
       "median": 0.0123856094301371,
       "n": 498
      },
      "psum": {
       "mean": 0.989810587577148,
       "median": 0.9981299561374187,
       "n": 498
      },
      "psum_bins": [
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       2,
       7,
       489
      ]
     }
    },
    "label": "Probabilities over plays, not sampled reasoning.",
    "margins": {
     "main": {
      "classes": {
       "none": {
        "close": {
         "den": 274,
         "n": 16,
         "share": 0.058394160583941604,
         "small": false
        },
        "eligible": 274,
        "risk_close": {
         "den": 16,
         "n": 0,
         "share": 0.0,
         "small": true
        }
       },
       "risk": {
        "close": {
         "den": 75,
         "n": 3,
         "share": 0.04,
         "small": true
        },
        "eligible": 75,
        "risk_close": {
         "den": 3,
         "n": 3,
         "share": 1.0,
         "small": true
        }
       },
       "safety": {
        "close": {
         "den": 178,
         "n": 10,
         "share": 0.056179775280898875,
         "small": true
        },
        "eligible": 178,
        "risk_close": {
         "den": 10,
         "n": 6,
         "share": 0.6,
         "small": true
        }
       }
      },
      "margin": 0.001,
      "missed": {
       "adj_close": 0.0,
       "adj_other": 0.8465608465608465,
       "adjusted": -0.8465608465608465,
       "adjusted_hi": -0.7,
       "adjusted_left_out": 85,
       "adjusted_lo": -1.0,
       "close": {
        "den": 3,
        "n": 0,
        "share": 0.0,
        "small": true
       },
       "covered": {
        "den": 3,
        "n": 3,
        "share": 1.0,
        "small": true
       },
       "crude": -0.8305084745762712,
       "crude_hi": -0.7450980392156863,
       "crude_left_out": 85,
       "crude_lo": -0.9090909090909091,
       "group": 62,
       "matches": 16,
       "na": null,
       "other": {
        "den": 59,
        "n": 49,
        "share": 0.8305084745762712,
        "small": true
       },
       "resamples": 2000,
       "strata": 8,
       "strata_both": 2
      },
      "stake": {
       "big": {
        "den": 38,
        "n": 1,
        "share": 0.02631578947368421,
        "small": true
       },
       "medium": {
        "den": 25,
        "n": 2,
        "share": 0.08,
        "small": true
       },
       "small": {
        "den": 12,
        "n": 0,
        "share": 0.0,
        "small": true
       }
      }
     },
     "robust": {
      "classes": {
       "none": {
        "close": {
         "den": 274,
         "n": 16,
         "share": 0.058394160583941604,
         "small": false
        },
        "eligible": 274,
        "risk_close": {
         "den": 16,
         "n": 0,
         "share": 0.0,
         "small": true
        }
       },
       "risk": {
        "close": {
         "den": 63,
         "n": 3,
         "share": 0.047619047619047616,
         "small": true
        },
        "eligible": 63,
        "risk_close": {
         "den": 3,
         "n": 3,
         "share": 1.0,
         "small": true
        }
       },
       "safety": {
        "close": {
         "den": 190,
         "n": 10,
         "share": 0.05263157894736842,
         "small": true
        },
        "eligible": 190,
        "risk_close": {
         "den": 10,
         "n": 6,
         "share": 0.6,
         "small": true
        }
       }
      },
      "margin": 0.02,
      "missed": {
       "adj_close": 0.0,
       "adj_other": 0.8465608465608465,
       "adjusted": -0.8465608465608465,
       "adjusted_hi": -0.7,
       "adjusted_left_out": 59,
       "adjusted_lo": -1.0,
       "close": {
        "den": 3,
        "n": 0,
        "share": 0.0,
        "small": true
       },
       "covered": {
        "den": 3,
        "n": 3,
        "share": 1.0,
        "small": true
       },
       "crude": -0.8125,
       "crude_hi": -0.7142857142857143,
       "crude_left_out": 59,
       "crude_lo": -0.9090909090909091,
       "group": 51,
       "matches": 16,
       "na": null,
       "other": {
        "den": 48,
        "n": 39,
        "share": 0.8125,
        "small": true
       },
       "resamples": 2000,
       "strata": 8,
       "strata_both": 2
      },
      "stake": {
       "big": {
        "den": 38,
        "n": 1,
        "share": 0.02631578947368421,
        "small": true
       },
       "medium": {
        "den": 25,
        "n": 2,
        "share": 0.08,
        "small": true
       },
       "small": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       }
      }
     }
    },
    "na": null,
    "risk_close": {
     "den": 29,
     "n": 9,
     "share": 0.3103448275862069,
     "small": true
    },
    "risk_close_of_eligible": {
     "den": 527,
     "n": 9,
     "share": 0.017077798861480076,
     "small": false
    },
    "risk_pair_other": {
     "den": 498,
     "n": 196,
     "share": 0.39357429718875503,
     "small": false
    },
    "ties": {
     "close": 4,
     "member": 13,
     "order": 4,
     "positive": 17,
     "top_two": 17,
     "zero": 0
    }
   },
   "confidence": {
    "auroc_best": 0.5155987644842074,
    "auroc_blunder": 0.511660999408634,
    "confident_badly_wrong": {
     "den": 1785,
     "n": 257,
     "share": 0.14397759103641455
    },
    "cutoffs": {
     "0.9": {
      "blunders": {
       "den": 1401,
       "n": 257,
       "share": 0.18344039971448964
      },
      "kept": {
       "den": 1785,
       "n": 1401,
       "share": 0.7848739495798319
      },
      "mean_loss": 0.051958098380077235,
      "too_few": false
     },
     "0.99": {
      "blunders": {
       "den": 766,
       "n": 167,
       "share": 0.21801566579634465
      },
      "kept": {
       "den": 1785,
       "n": 766,
       "share": 0.4291316526610644
      },
      "mean_loss": 0.06038407257736196,
      "too_few": false
     }
    },
    "ece": 0.3925087540780806,
    "held_out": {
     "blunders": 0.0805451483071452,
     "kept": 0.006494013390660502,
     "no_cutoff": 0,
     "runs": 300
    },
    "n": 1785,
    "surest": {
     "cut": 0.9959305047197058,
     "error_rate": 30.972245555596064,
     "n": 447,
     "too_few": false
    },
    "too_few": false
   },
   "quality": {
    "auroc": 0.5155987644842074,
    "best_share": 0.5017596782302665,
    "blunder_share": 0.21669180492709905,
    "brier": 0.4053580765463093,
    "clusters": 100,
    "confidence_n": 1785,
    "confident_badly_wrong": 0.14397759103641455,
    "cost_coverage": 1.0,
    "decisions": 1989,
    "ece": 0.3925087540780806,
    "engine_scored": 1989,
    "error_hi": 33.988169188371764,
    "error_lo": 27.84540792262547,
    "error_rate": 30.866173822236096,
    "first_date": "2026-10-19",
    "group_sizes": [
     14,
     15,
     16,
     17,
     18,
     19,
     20,
     21,
     22,
     23,
     24,
     25,
     26
    ],
    "grouped": 203,
    "invalid": 0,
    "last_date": "2026-10-26",
    "mean_confidence": 0.9179989501565119,
    "over_budget": 0.006033182503770739,
    "p95_loss": 0.310247254371643,
    "probability_source": "logprobs",
    "retry_mean": 6.685034188034188,
    "scored": 1989,
    "time_median": 1.746,
    "time_p90": 7.4812,
    "tokens_in": 4005.4695827048768,
    "tokens_out": 148.02362996480645,
    "too_few": false,
    "usd_per_1000": 2.9173718552036196,
    "very_confident": 0.7848739495798319,
    "worst_mean": 0.512480355573423
   },
   "risk": {
    "main": {
     "by_call": {
      "missed": 17.395758228783873,
      "needless": 1.8694416070595765,
      "none": 3.697974005196189,
      "right_risk": 1.7786702209528276,
      "right_safety": 3.9866154085397123,
      "wrong_risk": 2.137714351703918
     },
     "calls": {
      "missed": 422.0,
      "needless": 57.0,
      "right_risk": 205.0,
      "right_safety": 726.0,
      "wrong_risk": 41.0
     },
     "confidence": {
      "auroc_right_risk": 0.5162389735364876,
      "missed_mean_p": 0.9127431659425235,
      "missed_n": 373,
      "taken_grouped": 43,
      "taken_n": 259,
      "taken_other": 0
     },
     "error_rate": 30.866173822236096,
     "found": {
      "den": 668.0,
      "n": 205.0,
      "share": 0.3068862275449102
     },
     "good": {
      "den": 303.0,
      "n": 205.0,
      "share": 0.6765676567656765
     },
     "judgement": 61.704464633950494,
     "judgement_hi": 63.77670878438568,
     "judgement_lo": 59.67476963090035,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.08855291576673865,
      "x": 0.6931137724550899,
      "y": 0.07279693486590039
     },
     "margin": 0.001,
     "missed": {
      "den": 668.0,
      "n": 422.0,
      "share": 0.6317365269461078
     },
     "needless": {
      "den": 783.0,
      "n": 57.0,
      "share": 0.07279693486590039
     },
     "needless_by_loss": {
      "big": {
       "den": 783.0,
       "n": 35.0,
       "share": 0.04469987228607918
      },
      "medium": {
       "den": 783.0,
       "n": 15.0,
       "share": 0.019157088122605363
      },
      "small": {
       "den": 783.0,
       "n": 7.0,
       "share": 0.008939974457215836
      }
     },
     "no_trade_off": 538.0,
     "reference": null,
     "right_safety": {
      "den": 783.0,
      "n": 726.0,
      "share": 0.9272030651340997
     },
     "risk_pays": 668.0,
     "safety_pays": 783.0,
     "scored": 1989,
     "sdt": {
      "aim": {
       "den": 246.0,
       "n": 205.0,
       "share": 0.8333333333333334
      },
      "beta": 0.9325914640832854,
      "caution": 0.8936396642572635,
      "caution_hi": 0.9808948257322974,
      "caution_lo": 0.8107841143405541,
      "cstar": -0.06256707497373057,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.1154117000939414,
      "dprime_hi": 1.28761658966484,
      "dprime_lo": 0.9548056145107108,
      "false_alarm": {
       "den": 783.0,
       "n": 57.0,
       "share": 0.07279693486590039
      },
      "false_alarm_rate": 0.07334183673469388,
      "hit": {
       "den": 668.0,
       "n": 246.0,
       "share": 0.36826347305389223
      },
      "hit_rate": 0.3684603886397608,
      "missed_loss": 0.16398181572062145,
      "missed_n": 422.0,
      "na": null,
      "needless_loss": 0.13046734584005257,
      "needless_n": 57.0
     },
     "size": {
      "bad": {
       "mean": 7.459183673469388,
       "n": 98.0,
       "too_few": true
      },
      "right": {
       "mean": 8.770731707317074,
       "n": 205.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 383.0,
        "n": 141.0,
        "share": 0.3681462140992167
       },
       "missed": {
        "den": 383.0,
        "n": 224.0,
        "share": 0.5848563968668408
       },
       "risk_pays": 383.0,
       "wrong": {
        "den": 383.0,
        "n": 18.0,
        "share": 0.04699738903394256
       }
      },
      "medium": {
       "found": {
        "den": 196.0,
        "n": 56.0,
        "share": 0.2857142857142857
       },
       "missed": {
        "den": 196.0,
        "n": 127.0,
        "share": 0.6479591836734694
       },
       "risk_pays": 196.0,
       "wrong": {
        "den": 196.0,
        "n": 13.0,
        "share": 0.0663265306122449
       }
      },
      "small": {
       "found": {
        "den": 89.0,
        "n": 8.0,
        "share": 0.0898876404494382
       },
       "missed": {
        "den": 89.0,
        "n": 71.0,
        "share": 0.797752808988764
       },
       "risk_pays": 89.0,
       "wrong": {
        "den": 89.0,
        "n": 10.0,
        "share": 0.11235955056179775
       }
      }
     },
     "trade_off": 1451.0,
     "wrong": {
      "den": 668.0,
      "n": 41.0,
      "share": 0.061377245508982034
     }
    },
    "robust": {
     "by_call": {
      "missed": 17.019692057338176,
      "needless": 2.123590613082763,
      "none": 3.697974005196189,
      "right_risk": 1.275962848888683,
      "right_safety": 4.36268157998541,
      "wrong_risk": 2.3862727177448764
     },
     "calls": {
      "missed": 351.0,
      "needless": 75.0,
      "right_risk": 184.0,
      "right_safety": 797.0,
      "wrong_risk": 44.0
     },
     "confidence": {
      "auroc_right_risk": 0.5215831787260359,
      "missed_mean_p": 0.9116117941773004,
      "missed_n": 310,
      "taken_grouped": 43,
      "taken_n": 259,
      "taken_other": 0
     },
     "error_rate": 30.866173822236096,
     "found": {
      "den": 579.0,
      "n": 184.0,
      "share": 0.3177892918825561
     },
     "good": {
      "den": 303.0,
      "n": 184.0,
      "share": 0.6072607260726073
     },
     "judgement": 61.58900587853148,
     "judgement_hi": 63.862175897292104,
     "judgement_lo": 59.46976144238723,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.11139240506329114,
      "x": 0.6822107081174439,
      "y": 0.0860091743119266
     },
     "margin": 0.02,
     "missed": {
      "den": 579.0,
      "n": 351.0,
      "share": 0.6062176165803109
     },
     "needless": {
      "den": 872.0,
      "n": 75.0,
      "share": 0.0860091743119266
     },
     "needless_by_loss": {
      "big": {
       "den": 872.0,
       "n": 39.0,
       "share": 0.04472477064220184
      },
      "medium": {
       "den": 872.0,
       "n": 20.0,
       "share": 0.022935779816513763
      },
      "small": {
       "den": 872.0,
       "n": 16.0,
       "share": 0.01834862385321101
      }
     },
     "no_trade_off": 538.0,
     "reference": null,
     "right_safety": {
      "den": 872.0,
      "n": 797.0,
      "share": 0.9139908256880734
     },
     "risk_pays": 579.0,
     "safety_pays": 872.0,
     "scored": 1989,
     "sdt": {
      "aim": {
       "den": 228.0,
       "n": 184.0,
       "share": 0.8070175438596491
      },
      "beta": 0.8794332331015046,
      "caution": 0.8158654509309825,
      "caution_hi": 0.8948023232139332,
      "caution_lo": 0.7424148222663073,
      "cstar": -0.11746693759384165,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.09373441520436,
      "dprime_hi": 1.2734631960194358,
      "dprime_lo": 0.9113432709657523,
      "false_alarm": {
       "den": 872.0,
       "n": 75.0,
       "share": 0.0860091743119266
      },
      "false_alarm_rate": 0.08648339060710195,
      "hit": {
       "den": 579.0,
       "n": 228.0,
       "share": 0.39378238341968913
      },
      "hit_rate": 0.3939655172413793,
      "missed_loss": 0.19288984331649933,
      "missed_n": 351.0,
      "na": null,
      "needless_loss": 0.11263524611790975,
      "needless_n": 75.0
     },
     "size": {
      "bad": {
       "mean": 8.033613445378151,
       "n": 119.0,
       "too_few": true
      },
      "right": {
       "mean": 8.548913043478262,
       "n": 184.0,
       "too_few": true
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 383.0,
        "n": 136.0,
        "share": 0.35509138381201044
       },
       "missed": {
        "den": 383.0,
        "n": 224.0,
        "share": 0.5848563968668408
       },
       "risk_pays": 383.0,
       "wrong": {
        "den": 383.0,
        "n": 23.0,
        "share": 0.06005221932114883
       }
      },
      "medium": {
       "found": {
        "den": 196.0,
        "n": 48.0,
        "share": 0.24489795918367346
       },
       "missed": {
        "den": 196.0,
        "n": 127.0,
        "share": 0.6479591836734694
       },
       "risk_pays": 196.0,
       "wrong": {
        "den": 196.0,
        "n": 21.0,
        "share": 0.10714285714285714
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 1451.0,
     "wrong": {
      "den": 579.0,
      "n": 44.0,
      "share": 0.07599309153713299
     }
    }
   },
   "slug": "glm-5-3-on"
  },
  "GNUbg weak": {
   "close": {
    "decisions": 2136,
    "definition": "A decision is a close choice when the second most likely play has at least half the probability of the first (internally: split).",
    "eligible": 0,
    "exclusions": {
     "no_probabilities": 2136
    },
    "label": "Probabilities over plays, not sampled reasoning.",
    "na": "N/A: no full probability distribution"
   },
   "confidence": null,
   "quality": {
    "best_share": 0.46207865168539325,
    "blunder_share": 0.22144194756554308,
    "clusters": 94,
    "cost_coverage": 0.0,
    "decisions": 2136,
    "engine_scored": 2136,
    "error_hi": 29.79615872591371,
    "error_lo": 24.068358974977734,
    "error_rate": 26.975039085756972,
    "first_date": "2026-10-16",
    "group_sizes": [],
    "grouped": 0,
    "invalid": 0,
    "last_date": "2026-10-25",
    "over_budget": 0.0,
    "p95_loss": 0.26279696822166443,
    "probability_source": null,
    "retry_mean": 9.363295880149813e-06,
    "scored": 2136,
    "time_median": 0.01,
    "time_p90": 0.01,
    "too_few": false,
    "worst_mean": 0.38398785457432827
   },
   "risk": {
    "main": {
     "by_call": {
      "missed": 2.1905329022626305,
      "needless": 12.054322992036406,
      "none": 1.800976972678181,
      "right_risk": 2.929300579071491,
      "right_safety": 1.966152772698063,
      "wrong_risk": 6.033752867010202
     },
     "calls": {
      "missed": 111.0,
      "needless": 344.0,
      "right_risk": 483.0,
      "right_safety": 558.0,
      "wrong_risk": 176.0
     },
     "confidence": null,
     "error_rate": 26.975039085756972,
     "found": {
      "den": 770.0,
      "n": 483.0,
      "share": 0.6272727272727273
     },
     "good": {
      "den": 1003.0,
      "n": 483.0,
      "share": 0.481555333998006
     },
     "judgement": 62.2949002217295,
     "judgement_hi": 64.83975461405096,
     "judgement_lo": 59.54184951561016,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.6132404181184669,
      "x": 0.37272727272727274,
      "y": 0.38137472283813745
     },
     "margin": 0.001,
     "missed": {
      "den": 770.0,
      "n": 111.0,
      "share": 0.14415584415584415
     },
     "needless": {
      "den": 902.0,
      "n": 344.0,
      "share": 0.38137472283813745
     },
     "needless_by_loss": {
      "big": {
       "den": 902.0,
       "n": 211.0,
       "share": 0.23392461197339245
      },
      "medium": {
       "den": 902.0,
       "n": 100.0,
       "share": 0.11086474501108648
      },
      "small": {
       "den": 902.0,
       "n": 33.0,
       "share": 0.036585365853658534
      }
     },
     "no_trade_off": 464.0,
     "reference": null,
     "right_safety": {
      "den": 902.0,
      "n": 558.0,
      "share": 0.6186252771618626
     },
     "risk_pays": 770.0,
     "safety_pays": 902.0,
     "scored": 2136,
     "sdt": {
      "aim": {
       "den": 659.0,
       "n": 483.0,
       "share": 0.7329286798179059
      },
      "beta": 2.0800481670598545,
      "caution": -0.3791371028282712,
      "caution_hi": -0.30185885864764,
      "caution_lo": -0.46485664868300214,
      "cstar": 0.5379969108507809,
      "cstar_note": "unstable: fewer than 200 missed risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.3613294721845584,
      "dprime_hi": 1.504102077679335,
      "dprime_lo": 1.2229762963863378,
      "false_alarm": {
       "den": 902.0,
       "n": 344.0,
       "share": 0.38137472283813745
      },
      "false_alarm_rate": 0.3815060908084164,
      "hit": {
       "den": 770.0,
       "n": 659.0,
       "share": 0.8558441558441559
      },
      "hit_rate": 0.8553826199740596,
      "missed_loss": 0.08430591494113475,
      "missed_n": 111.0,
      "na": null,
      "needless_loss": 0.14969787157552186,
      "needless_n": 344.0
     },
     "size": {
      "bad": {
       "mean": 13.467307692307692,
       "n": 520.0,
       "too_few": false
      },
      "right": {
       "mean": 11.200828157349896,
       "n": 483.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 397.0,
        "n": 327.0,
        "share": 0.8236775818639799
       },
       "missed": {
        "den": 397.0,
        "n": 30.0,
        "share": 0.07556675062972293
       },
       "risk_pays": 397.0,
       "wrong": {
        "den": 397.0,
        "n": 40.0,
        "share": 0.10075566750629723
       }
      },
      "medium": {
       "found": {
        "den": 251.0,
        "n": 124.0,
        "share": 0.4940239043824701
       },
       "missed": {
        "den": 251.0,
        "n": 40.0,
        "share": 0.1593625498007968
       },
       "risk_pays": 251.0,
       "wrong": {
        "den": 251.0,
        "n": 87.0,
        "share": 0.3466135458167331
       }
      },
      "small": {
       "found": {
        "den": 122.0,
        "n": 32.0,
        "share": 0.26229508196721313
       },
       "missed": {
        "den": 122.0,
        "n": 41.0,
        "share": 0.3360655737704918
       },
       "risk_pays": 122.0,
       "wrong": {
        "den": 122.0,
        "n": 49.0,
        "share": 0.4016393442622951
       }
      }
     },
     "trade_off": 1672.0,
     "wrong": {
      "den": 770.0,
      "n": 176.0,
      "share": 0.22857142857142856
     }
    },
    "robust": {
     "by_call": {
      "missed": 2.0029648147830357,
      "needless": 12.91498454629482,
      "none": 1.800976972678181,
      "right_risk": 2.5024489335687865,
      "right_safety": 2.153720860177658,
      "wrong_risk": 5.599942958254493
     },
     "calls": {
      "missed": 70.0,
      "needless": 425.0,
      "right_risk": 420.0,
      "right_safety": 599.0,
      "wrong_risk": 158.0
     },
     "confidence": null,
     "error_rate": 26.975039085756972,
     "found": {
      "den": 648.0,
      "n": 420.0,
      "share": 0.6481481481481481
     },
     "good": {
      "den": 1003.0,
      "n": 420.0,
      "share": 0.4187437686939183
     },
     "judgement": 61.655454282407405,
     "judgement_hi": 64.60633344061868,
     "judgement_lo": 58.72897035579289,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.6929824561403509,
      "x": 0.35185185185185186,
      "y": 0.4150390625
     },
     "margin": 0.02,
     "missed": {
      "den": 648.0,
      "n": 70.0,
      "share": 0.10802469135802469
     },
     "needless": {
      "den": 1024.0,
      "n": 425.0,
      "share": 0.4150390625
     },
     "needless_by_loss": {
      "big": {
       "den": 1024.0,
       "n": 225.0,
       "share": 0.2197265625
      },
      "medium": {
       "den": 1024.0,
       "n": 126.0,
       "share": 0.123046875
      },
      "small": {
       "den": 1024.0,
       "n": 74.0,
       "share": 0.072265625
      }
     },
     "no_trade_off": 464.0,
     "reference": null,
     "right_safety": {
      "den": 1024.0,
      "n": 599.0,
      "share": 0.5849609375
     },
     "risk_pays": 648.0,
     "safety_pays": 1024.0,
     "scored": 2136,
     "sdt": {
      "aim": {
       "den": 578.0,
       "n": 420.0,
       "share": 0.726643598615917
      },
      "beta": 1.678242176422712,
      "caution": -0.5097326098139878,
      "caution_hi": -0.4202863348236549,
      "caution_lo": -0.6035560508209965,
      "cstar": 0.35750009617271256,
      "cstar_note": "unstable: fewer than 200 missed risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.4482427490830623,
      "dprime_hi": 1.6149286759690253,
      "dprime_lo": 1.3105725087001423,
      "false_alarm": {
       "den": 1024.0,
       "n": 425.0,
       "share": 0.4150390625
      },
      "false_alarm_rate": 0.4151219512195122,
      "hit": {
       "den": 648.0,
       "n": 578.0,
       "share": 0.8919753086419753
      },
      "hit_rate": 0.8913713405238829,
      "missed_loss": 0.12223808126790182,
      "missed_n": 70.0,
      "na": null,
      "needless_loss": 0.12981838583946229,
      "needless_n": 425.0
     },
     "size": {
      "bad": {
       "mean": 13.212692967409948,
       "n": 583.0,
       "too_few": false
      },
      "right": {
       "mean": 11.214285714285714,
       "n": 420.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 397.0,
        "n": 316.0,
        "share": 0.7959697732997482
       },
       "missed": {
        "den": 397.0,
        "n": 30.0,
        "share": 0.07556675062972293
       },
       "risk_pays": 397.0,
       "wrong": {
        "den": 397.0,
        "n": 51.0,
        "share": 0.12846347607052896
       }
      },
      "medium": {
       "found": {
        "den": 251.0,
        "n": 104.0,
        "share": 0.41434262948207173
       },
       "missed": {
        "den": 251.0,
        "n": 40.0,
        "share": 0.1593625498007968
       },
       "risk_pays": 251.0,
       "wrong": {
        "den": 251.0,
        "n": 107.0,
        "share": 0.4262948207171315
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 1672.0,
     "wrong": {
      "den": 648.0,
      "n": 158.0,
      "share": 0.24382716049382716
     }
    }
   },
   "slug": "gnubg-weak"
  },
  "Gemini 3.8 Flash (high)": {
   "close": {
    "decisions": 2168,
    "definition": "A decision is a close choice when the second most likely play has at least half the probability of the first (internally: split).",
    "eligible": 0,
    "exclusions": {
     "no_probabilities": 1974,
     "not_audited": 194
    },
    "label": "Probabilities over plays, not sampled reasoning.",
    "na": "N/A: no full probability distribution"
   },
   "confidence": null,
   "quality": {
    "best_share": 0.6314575645756457,
    "blunder_share": 0.10977859778597786,
    "clusters": 104,
    "cost_coverage": 1.0,
    "decisions": 2168,
    "engine_scored": 2168,
    "error_hi": 17.0154381065425,
    "error_lo": 12.903810956625556,
    "error_rate": 14.946429347585049,
    "first_date": "2026-10-19",
    "group_sizes": [
     14,
     15,
     16,
     17,
     18,
     19,
     20,
     21,
     22,
     23,
     24,
     25,
     26
    ],
    "grouped": 237,
    "invalid": 0,
    "last_date": "2026-10-26",
    "over_budget": 0.18404059040590406,
    "p95_loss": 0.1668665647506714,
    "probability_source": "choice_only",
    "retry_mean": 2.29165036900369,
    "scored": 2168,
    "time_median": 9.5885,
    "time_p90": 44.36469999999999,
    "tokens_in": 4344.454335793358,
    "tokens_out": 5091.751383763838,
    "too_few": false,
    "usd_per_1000": 22.35240844095941,
    "worst_mean": 0.30519890592054083
   },
   "risk": {
    "main": {
     "by_call": {
      "missed": 10.769729938949165,
      "needless": 0.41545713079811464,
      "none": 0.5035740615476981,
      "right_risk": 1.1646454445880277,
      "right_safety": 1.5018426940339957,
      "wrong_risk": 0.5911800776680457
     },
     "calls": {
      "missed": 450.0,
      "needless": 21.0,
      "right_risk": 264.0,
      "right_safety": 858.0,
      "wrong_risk": 17.0
     },
     "confidence": null,
     "error_rate": 14.946429347585049,
     "found": {
      "den": 731.0,
      "n": 264.0,
      "share": 0.36114911080711354
     },
     "good": {
      "den": 302.0,
      "n": 264.0,
      "share": 0.8741721854304636
     },
     "judgement": 66.86291629120892,
     "judgement_hi": 68.52115491308702,
     "judgement_lo": 65.21442056759784,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.03640256959314775,
      "x": 0.6388508891928865,
      "y": 0.023890784982935155
     },
     "margin": 0.001,
     "missed": {
      "den": 731.0,
      "n": 450.0,
      "share": 0.615595075239398
     },
     "needless": {
      "den": 879.0,
      "n": 21.0,
      "share": 0.023890784982935155
     },
     "needless_by_loss": {
      "big": {
       "den": 879.0,
       "n": 4.0,
       "share": 0.004550625711035267
      },
      "medium": {
       "den": 879.0,
       "n": 9.0,
       "share": 0.010238907849829351
      },
      "small": {
       "den": 879.0,
       "n": 8.0,
       "share": 0.009101251422070534
      }
     },
     "no_trade_off": 558.0,
     "reference": null,
     "right_safety": {
      "den": 879.0,
      "n": 858.0,
      "share": 0.9761092150170648
     },
     "risk_pays": 731.0,
     "safety_pays": 879.0,
     "scored": 2168,
     "sdt": {
      "aim": {
       "den": 281.0,
       "n": 264.0,
       "share": 0.9395017793594306
      },
      "beta": 0.9939993984020494,
      "caution": 1.131649162847859,
      "caution_hi": 1.233981732895945,
      "caution_lo": 1.0472576390650172,
      "cstar": -0.0035905375457885385,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.6762608601968825,
      "dprime_hi": 1.8793164512548597,
      "dprime_lo": 1.4939467752384923,
      "false_alarm": {
       "den": 879.0,
       "n": 21.0,
       "share": 0.023890784982935155
      },
      "false_alarm_rate": 0.024431818181818183,
      "hit": {
       "den": 731.0,
       "n": 281.0,
       "share": 0.3844049247606019
      },
      "hit_rate": 0.38456284153005466,
      "missed_loss": 0.10377233114507463,
      "missed_n": 450.0,
      "na": null,
      "needless_loss": 0.0857820056733631,
      "needless_n": 21.0
     },
     "size": {
      "bad": {
       "mean": 7.447368421052632,
       "n": 38.0,
       "too_few": true
      },
      "right": {
       "mean": 9.227272727272727,
       "n": 264.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 365.0,
        "n": 193.0,
        "share": 0.5287671232876713
       },
       "missed": {
        "den": 365.0,
        "n": 167.0,
        "share": 0.4575342465753425
       },
       "risk_pays": 365.0,
       "wrong": {
        "den": 365.0,
        "n": 5.0,
        "share": 0.0136986301369863
       }
      },
      "medium": {
       "found": {
        "den": 245.0,
        "n": 57.0,
        "share": 0.23265306122448978
       },
       "missed": {
        "den": 245.0,
        "n": 180.0,
        "share": 0.7346938775510204
       },
       "risk_pays": 245.0,
       "wrong": {
        "den": 245.0,
        "n": 8.0,
        "share": 0.0326530612244898
       }
      },
      "small": {
       "found": {
        "den": 121.0,
        "n": 14.0,
        "share": 0.11570247933884298
       },
       "missed": {
        "den": 121.0,
        "n": 103.0,
        "share": 0.8512396694214877
       },
       "risk_pays": 121.0,
       "wrong": {
        "den": 121.0,
        "n": 4.0,
        "share": 0.03305785123966942
       }
      }
     },
     "trade_off": 1610.0,
     "wrong": {
      "den": 731.0,
      "n": 17.0,
      "share": 0.023255813953488372
     }
    },
    "robust": {
     "by_call": {
      "missed": 10.253361835250995,
      "needless": 0.4444622410619391,
      "none": 0.5035740615476981,
      "right_risk": 1.0218876053156447,
      "right_safety": 2.018210797732167,
      "wrong_risk": 0.7049328066766042
     },
     "calls": {
      "missed": 347.0,
      "needless": 39.0,
      "right_risk": 239.0,
      "right_safety": 961.0,
      "wrong_risk": 24.0
     },
     "confidence": null,
     "error_rate": 14.946429347585049,
     "found": {
      "den": 610.0,
      "n": 239.0,
      "share": 0.3918032786885246
     },
     "good": {
      "den": 302.0,
      "n": 239.0,
      "share": 0.7913907284768212
     },
     "judgement": 67.64016393442623,
     "judgement_hi": 69.63751236568436,
     "judgement_lo": 65.53290391788272,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.0646900269541779,
      "x": 0.6081967213114754,
      "y": 0.039
     },
     "margin": 0.02,
     "missed": {
      "den": 610.0,
      "n": 347.0,
      "share": 0.5688524590163935
     },
     "needless": {
      "den": 1000.0,
      "n": 39.0,
      "share": 0.039
     },
     "needless_by_loss": {
      "big": {
       "den": 1000.0,
       "n": 4.0,
       "share": 0.004
      },
      "medium": {
       "den": 1000.0,
       "n": 12.0,
       "share": 0.012
      },
      "small": {
       "den": 1000.0,
       "n": 23.0,
       "share": 0.023
      }
     },
     "no_trade_off": 558.0,
     "reference": null,
     "right_safety": {
      "den": 1000.0,
      "n": 961.0,
      "share": 0.961
     },
     "risk_pays": 610.0,
     "safety_pays": 1000.0,
     "scored": 2168,
     "sdt": {
      "aim": {
       "den": 263.0,
       "n": 239.0,
       "share": 0.908745247148289
      },
      "beta": 0.6322715213337458,
      "caution": 0.9650737201918687,
      "caution_hi": 1.0506275306303416,
      "caution_lo": 0.892254021606742,
      "cstar": -0.28945084363233703,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.5838141944244297,
      "dprime_hi": 1.7902340192655846,
      "dprime_lo": 1.4020988912206782,
      "false_alarm": {
       "den": 1000.0,
       "n": 39.0,
       "share": 0.039
      },
      "false_alarm_rate": 0.039460539460539464,
      "hit": {
       "den": 610.0,
       "n": 263.0,
       "share": 0.4311475409836066
      },
      "hit_rate": 0.43126022913256956,
      "missed_loss": 0.1281227000508597,
      "missed_n": 347.0,
      "na": null,
      "needless_loss": 0.049415084031912,
      "needless_n": 39.0
     },
     "size": {
      "bad": {
       "mean": 8.984126984126984,
       "n": 63.0,
       "too_few": true
      },
      "right": {
       "mean": 9.00836820083682,
       "n": 239.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 365.0,
        "n": 190.0,
        "share": 0.5205479452054794
       },
       "missed": {
        "den": 365.0,
        "n": 167.0,
        "share": 0.4575342465753425
       },
       "risk_pays": 365.0,
       "wrong": {
        "den": 365.0,
        "n": 8.0,
        "share": 0.021917808219178082
       }
      },
      "medium": {
       "found": {
        "den": 245.0,
        "n": 49.0,
        "share": 0.2
       },
       "missed": {
        "den": 245.0,
        "n": 180.0,
        "share": 0.7346938775510204
       },
       "risk_pays": 245.0,
       "wrong": {
        "den": 245.0,
        "n": 16.0,
        "share": 0.0653061224489796
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 1610.0,
     "wrong": {
      "den": 610.0,
      "n": 24.0,
      "share": 0.03934426229508197
     }
    }
   },
   "slug": "gemini-3-8-flash-high"
  },
  "Gemini 3.8 Flash (low)": {
   "close": {
    "decisions": 2053,
    "definition": "A decision is a close choice when the second most likely play has at least half the probability of the first (internally: split).",
    "eligible": 0,
    "exclusions": {
     "no_probabilities": 1830,
     "not_audited": 223
    },
    "label": "Probabilities over plays, not sampled reasoning.",
    "na": "N/A: no full probability distribution"
   },
   "confidence": null,
   "quality": {
    "best_share": 0.5903555772040916,
    "blunder_share": 0.13492450073063808,
    "clusters": 102,
    "cost_coverage": 1.0,
    "decisions": 2053,
    "engine_scored": 2053,
    "error_hi": 18.16125630250965,
    "error_lo": 14.498559950292867,
    "error_rate": 16.332104504543924,
    "first_date": "2026-10-19",
    "group_sizes": [
     14,
     15,
     16,
     17,
     18,
     19,
     20,
     21,
     22,
     23,
     24,
     25,
     26
    ],
    "grouped": 202,
    "invalid": 0,
    "last_date": "2026-10-26",
    "over_budget": 0.007306380905991232,
    "p95_loss": 0.1801735401153564,
    "probability_source": "choice_only",
    "retry_mean": 0.04003701899659035,
    "scored": 2053,
    "time_median": 2.158,
    "time_p90": 6.873399999999998,
    "tokens_in": 4311.187043351193,
    "tokens_out": 237.11885046273747,
    "too_few": false,
    "usd_per_1000": 4.12258597174866,
    "worst_mean": 0.27874858865460145
   },
   "risk": {
    "main": {
     "by_call": {
      "missed": 9.900604196426292,
      "needless": 0.7987130983132475,
      "none": 1.2852092138802313,
      "right_risk": 0.7850529795673146,
      "right_safety": 2.819080992669404,
      "wrong_risk": 0.7434440236874342
     },
     "calls": {
      "missed": 400.0,
      "needless": 32.0,
      "right_risk": 214.0,
      "right_safety": 877.0,
      "wrong_risk": 19.0
     },
     "confidence": null,
     "error_rate": 16.332104504543924,
     "found": {
      "den": 633.0,
      "n": 214.0,
      "share": 0.3380726698262243
     },
     "good": {
      "den": 265.0,
      "n": 214.0,
      "share": 0.8075471698113208
     },
     "judgement": 65.14345747370946,
     "judgement_hi": 66.81284985583373,
     "judgement_lo": 63.52906966652296,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.045346062052505964,
      "x": 0.6619273301737757,
      "y": 0.0352035203520352
     },
     "margin": 0.001,
     "missed": {
      "den": 633.0,
      "n": 400.0,
      "share": 0.631911532385466
     },
     "needless": {
      "den": 909.0,
      "n": 32.0,
      "share": 0.0352035203520352
     },
     "needless_by_loss": {
      "big": {
       "den": 909.0,
       "n": 15.0,
       "share": 0.0165016501650165
      },
      "medium": {
       "den": 909.0,
       "n": 10.0,
       "share": 0.011001100110011002
      },
      "small": {
       "den": 909.0,
       "n": 7.0,
       "share": 0.007700770077007701
      }
     },
     "no_trade_off": 511.0,
     "reference": null,
     "right_safety": {
      "den": 909.0,
      "n": 877.0,
      "share": 0.9647964796479648
     },
     "risk_pays": 633.0,
     "safety_pays": 909.0,
     "scored": 2053,
     "sdt": {
      "aim": {
       "den": 233.0,
       "n": 214.0,
       "share": 0.9184549356223176
      },
      "beta": 1.4481024691810829,
      "caution": 1.069555755854926,
      "caution_hi": 1.1698498196372158,
      "caution_lo": 0.9874159439086214,
      "cstar": 0.252496217441624,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.4663746697685283,
      "dprime_hi": 1.639246837317353,
      "dprime_lo": 1.3250812403208396,
      "false_alarm": {
       "den": 909.0,
       "n": 32.0,
       "share": 0.0352035203520352
      },
      "false_alarm_rate": 0.03571428571428571,
      "hit": {
       "den": 633.0,
       "n": 233.0,
       "share": 0.36808846761453395
      },
      "hit_rate": 0.36829652996845424,
      "missed_loss": 0.10162970207631589,
      "missed_n": 400.0,
      "na": null,
      "needless_loss": 0.10248487442731857,
      "needless_n": 32.0
     },
     "size": {
      "bad": {
       "mean": 7.03921568627451,
       "n": 51.0,
       "too_few": true
      },
      "right": {
       "mean": 9.705607476635514,
       "n": 214.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 305.0,
        "n": 151.0,
        "share": 0.49508196721311476
       },
       "missed": {
        "den": 305.0,
        "n": 148.0,
        "share": 0.4852459016393443
       },
       "risk_pays": 305.0,
       "wrong": {
        "den": 305.0,
        "n": 6.0,
        "share": 0.019672131147540985
       }
      },
      "medium": {
       "found": {
        "den": 235.0,
        "n": 58.0,
        "share": 0.24680851063829787
       },
       "missed": {
        "den": 235.0,
        "n": 167.0,
        "share": 0.7106382978723405
       },
       "risk_pays": 235.0,
       "wrong": {
        "den": 235.0,
        "n": 10.0,
        "share": 0.0425531914893617
       }
      },
      "small": {
       "found": {
        "den": 93.0,
        "n": 5.0,
        "share": 0.053763440860215055
       },
       "missed": {
        "den": 93.0,
        "n": 85.0,
        "share": 0.9139784946236559
       },
       "risk_pays": 93.0,
       "wrong": {
        "den": 93.0,
        "n": 3.0,
        "share": 0.03225806451612903
       }
      }
     },
     "trade_off": 1542.0,
     "wrong": {
      "den": 633.0,
      "n": 19.0,
      "share": 0.030015797788309637
     }
    },
    "robust": {
     "by_call": {
      "missed": 9.517085614636416,
      "needless": 0.9102684895236262,
      "none": 1.2852092138802313,
      "right_risk": 0.6883284462642159,
      "right_safety": 3.2025995744592786,
      "wrong_risk": 0.7286131657801543
     },
     "calls": {
      "missed": 315.0,
      "needless": 40.0,
      "right_risk": 203.0,
      "right_safety": 962.0,
      "wrong_risk": 22.0
     },
     "confidence": null,
     "error_rate": 16.332104504543924,
     "found": {
      "den": 540.0,
      "n": 203.0,
      "share": 0.37592592592592594
     },
     "good": {
      "den": 265.0,
      "n": 203.0,
      "share": 0.7660377358490567
     },
     "judgement": 66.80028831226437,
     "judgement_hi": 68.76648341024244,
     "judgement_lo": 64.8715311230423,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.06528189910979229,
      "x": 0.6240740740740741,
      "y": 0.03992015968063872
     },
     "margin": 0.02,
     "missed": {
      "den": 540.0,
      "n": 315.0,
      "share": 0.5833333333333334
     },
     "needless": {
      "den": 1002.0,
      "n": 40.0,
      "share": 0.03992015968063872
     },
     "needless_by_loss": {
      "big": {
       "den": 1002.0,
       "n": 17.0,
       "share": 0.016966067864271458
      },
      "medium": {
       "den": 1002.0,
       "n": 11.0,
       "share": 0.010978043912175649
      },
      "small": {
       "den": 1002.0,
       "n": 12.0,
       "share": 0.011976047904191617
      }
     },
     "no_trade_off": 511.0,
     "reference": null,
     "right_safety": {
      "den": 1002.0,
      "n": 962.0,
      "share": 0.9600798403193613
     },
     "risk_pays": 540.0,
     "safety_pays": 1002.0,
     "scored": 2053,
     "sdt": {
      "aim": {
       "den": 225.0,
       "n": 203.0,
       "share": 0.9022222222222223
      },
      "beta": 1.3976230583349796,
      "caution": 0.9781700211810852,
      "caution_hi": 1.07476115013689,
      "caution_lo": 0.8981692490417221,
      "cstar": 0.21791246554679022,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.536272729437686,
      "dprime_hi": 1.709713268925977,
      "dprime_lo": 1.381400580683983,
      "false_alarm": {
       "den": 1002.0,
       "n": 40.0,
       "share": 0.03992015968063872
      },
      "false_alarm_rate": 0.04037886340977069,
      "hit": {
       "den": 540.0,
       "n": 225.0,
       "share": 0.4166666666666667
      },
      "hit_rate": 0.4168207024029575,
      "missed_loss": 0.12405445566253057,
      "missed_n": 315.0,
      "na": null,
      "needless_loss": 0.09343906044960022,
      "needless_n": 40.0
     },
     "size": {
      "bad": {
       "mean": 7.661290322580645,
       "n": 62.0,
       "too_few": true
      },
      "right": {
       "mean": 9.660098522167488,
       "n": 203.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 305.0,
        "n": 149.0,
        "share": 0.4885245901639344
       },
       "missed": {
        "den": 305.0,
        "n": 148.0,
        "share": 0.4852459016393443
       },
       "risk_pays": 305.0,
       "wrong": {
        "den": 305.0,
        "n": 8.0,
        "share": 0.02622950819672131
       }
      },
      "medium": {
       "found": {
        "den": 235.0,
        "n": 54.0,
        "share": 0.2297872340425532
       },
       "missed": {
        "den": 235.0,
        "n": 167.0,
        "share": 0.7106382978723405
       },
       "risk_pays": 235.0,
       "wrong": {
        "den": 235.0,
        "n": 14.0,
        "share": 0.059574468085106386
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 1542.0,
     "wrong": {
      "den": 540.0,
      "n": 22.0,
      "share": 0.040740740740740744
     }
    }
   },
   "slug": "gemini-3-8-flash-low"
  },
  "Gemma 4 31B (off)": {
   "close": {
    "close": {
     "den": 475,
     "n": 14,
     "share": 0.029473684210526315,
     "small": false
    },
    "decisions": 1847,
    "definition": "A decision is a close choice when the second most likely play has at least half the probability of the first (internally: split).",
    "eligible": 475,
    "exclusions": {
     "grouped": 148,
     "not_audited": 231,
     "sparse": 993
    },
    "groups": {
     "close": {
      "by_count": [
       6,
       5,
       3,
       0,
       0
      ],
      "differ": {
       "hitting": {
        "den": 14,
        "n": 6,
        "share": 0.42857142857142855,
        "small": true
       },
       "points": {
        "den": 14,
        "n": 2,
        "share": 0.14285714285714285,
        "small": true
       },
       "prime": {
        "den": 14,
        "n": 3,
        "share": 0.21428571428571427,
        "small": true
       },
       "race": {
        "den": 14,
        "n": 0,
        "share": 0.0,
        "small": true
       }
      },
      "gap": {
       "mean": 0.07922504203660148,
       "median": 0.0398486852645874,
       "n": 14
      },
      "loss1": {
       "mean": 0.028444605214255198,
       "median": 0.0,
       "n": 14
      },
      "loss2": {
       "mean": 0.059581445796149116,
       "median": 0.013606369495391846,
       "n": 14
      },
      "n": 14,
      "p1": {
       "mean": 0.5635302120792763,
       "median": 0.5612691114520361,
       "n": 14
      },
      "p2": {
       "mean": 0.40770560642873044,
       "median": 0.4148040549600782,
       "n": 14
      },
      "psum": {
       "mean": 0.9712358185080069,
       "median": 0.9984950397994548,
       "n": 14
      },
      "psum_bins": [
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       2,
       12
      ]
     },
     "other": {
      "by_count": [
       185,
       178,
       81,
       17,
       0
      ],
      "differ": {
       "hitting": {
        "den": 461,
        "n": 162,
        "share": 0.351409978308026,
        "small": false
       },
       "points": {
        "den": 461,
        "n": 65,
        "share": 0.14099783080260303,
        "small": false
       },
       "prime": {
        "den": 461,
        "n": 116,
        "share": 0.25162689804772237,
        "small": false
       },
       "race": {
        "den": 461,
        "n": 48,
        "share": 0.10412147505422993,
        "small": false
       }
      },
      "gap": {
       "mean": 0.09990072108142546,
       "median": 0.03555893898010254,
       "n": 461
      },
      "loss1": {
       "mean": 0.02353126767400547,
       "median": 0.0,
       "n": 461
      },
      "loss2": {
       "mean": 0.09509843911110967,
       "median": 0.029202938079833984,
       "n": 461
      },
      "n": 461,
      "p1": {
       "mean": 0.9768127994348579,
       "median": 0.999836834219735,
       "n": 461
      },
      "p2": {
       "mean": 0.02118637113396625,
       "median": 0.00015256402578962944,
       "n": 461
      },
      "psum": {
       "mean": 0.9979991705688241,
       "median": 0.9999992487647918,
       "n": 461
      },
      "psum_bins": [
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       2,
       459
      ]
     }
    },
    "label": "Probabilities over plays, not sampled reasoning.",
    "margins": {
     "main": {
      "classes": {
       "none": {
        "close": {
         "den": 250,
         "n": 6,
         "share": 0.024,
         "small": false
        },
        "eligible": 250,
        "risk_close": {
         "den": 6,
         "n": 0,
         "share": 0.0,
         "small": true
        }
       },
       "risk": {
        "close": {
         "den": 81,
         "n": 4,
         "share": 0.04938271604938271,
         "small": true
        },
        "eligible": 81,
        "risk_close": {
         "den": 4,
         "n": 2,
         "share": 0.5,
         "small": true
        }
       },
       "safety": {
        "close": {
         "den": 144,
         "n": 4,
         "share": 0.027777777777777776,
         "small": true
        },
        "eligible": 144,
        "risk_close": {
         "den": 4,
         "n": 4,
         "share": 1.0,
         "small": true
        }
       }
      },
      "margin": 0.001,
      "missed": {
       "adj_close": 0.0,
       "adj_other": 0.7115384615384616,
       "adjusted": -0.7115384615384616,
       "adjusted_hi": 0.0,
       "adjusted_left_out": 281,
       "adjusted_lo": -1.0,
       "close": {
        "den": 2,
        "n": 0,
        "share": 0.0,
        "small": true
       },
       "covered": {
        "den": 2,
        "n": 2,
        "share": 1.0,
        "small": true
       },
       "crude": -0.6363636363636364,
       "crude_hi": -0.5,
       "crude_left_out": 233,
       "crude_lo": -0.7457164869029276,
       "group": 57,
       "matches": 16,
       "na": null,
       "other": {
        "den": 55,
        "n": 35,
        "share": 0.6363636363636364,
        "small": true
       },
       "resamples": 2000,
       "strata": 11,
       "strata_both": 2
      },
      "stake": {
       "big": {
        "den": 40,
        "n": 1,
        "share": 0.025,
        "small": true
       },
       "medium": {
        "den": 29,
        "n": 2,
        "share": 0.06896551724137931,
        "small": true
       },
       "small": {
        "den": 12,
        "n": 1,
        "share": 0.08333333333333333,
        "small": true
       }
      }
     },
     "robust": {
      "classes": {
       "none": {
        "close": {
         "den": 250,
         "n": 6,
         "share": 0.024,
         "small": false
        },
        "eligible": 250,
        "risk_close": {
         "den": 6,
         "n": 0,
         "share": 0.0,
         "small": true
        }
       },
       "risk": {
        "close": {
         "den": 69,
         "n": 3,
         "share": 0.043478260869565216,
         "small": true
        },
        "eligible": 69,
        "risk_close": {
         "den": 3,
         "n": 2,
         "share": 0.6666666666666666,
         "small": true
        }
       },
       "safety": {
        "close": {
         "den": 156,
         "n": 5,
         "share": 0.03205128205128205,
         "small": true
        },
        "eligible": 156,
        "risk_close": {
         "den": 5,
         "n": 4,
         "share": 0.8,
         "small": true
        }
       }
      },
      "margin": 0.02,
      "missed": {
       "adj_close": 0.0,
       "adj_other": 0.6666666666666667,
       "adjusted": -0.6666666666666667,
       "adjusted_hi": 0.0,
       "adjusted_left_out": 350,
       "adjusted_lo": -1.0,
       "close": {
        "den": 2,
        "n": 0,
        "share": 0.0,
        "small": true
       },
       "covered": {
        "den": 2,
        "n": 2,
        "share": 1.0,
        "small": true
       },
       "crude": -0.574468085106383,
       "crude_hi": -0.43478260869565216,
       "crude_left_out": 239,
       "crude_lo": -0.6923076923076923,
       "group": 49,
       "matches": 16,
       "na": null,
       "other": {
        "den": 47,
        "n": 27,
        "share": 0.574468085106383,
        "small": true
       },
       "resamples": 2000,
       "strata": 10,
       "strata_both": 2
      },
      "stake": {
       "big": {
        "den": 40,
        "n": 1,
        "share": 0.025,
        "small": true
       },
       "medium": {
        "den": 29,
        "n": 2,
        "share": 0.06896551724137931,
        "small": true
       },
       "small": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       }
      }
     }
    },
    "na": null,
    "risk_close": {
     "den": 14,
     "n": 6,
     "share": 0.42857142857142855,
     "small": true
    },
    "risk_close_of_eligible": {
     "den": 475,
     "n": 6,
     "share": 0.01263157894736842,
     "small": false
    },
    "risk_pair_other": {
     "den": 461,
     "n": 154,
     "share": 0.33405639913232105,
     "small": false
    },
    "ties": {
     "close": 0,
     "member": 1,
     "order": 0,
     "positive": 1,
     "top_two": 1,
     "zero": 0
    }
   },
   "confidence": {
    "auroc_best": 0.6606563485365958,
    "auroc_blunder": 0.6031729825250344,
    "confident_badly_wrong": {
     "den": 1664,
     "n": 239,
     "share": 0.14362980769230768
    },
    "cutoffs": {
     "0.9": {
      "blunders": {
       "den": 1510,
       "n": 239,
       "share": 0.15827814569536425
      },
      "kept": {
       "den": 1664,
       "n": 1510,
       "share": 0.9074519230769231
      },
      "mean_loss": 0.039637134959366147,
      "too_few": false
     },
     "0.99": {
      "blunders": {
       "den": 1349,
       "n": 200,
       "share": 0.14825796886582654
      },
      "kept": {
       "den": 1664,
       "n": 1349,
       "share": 0.8106971153846154
      },
      "mean_loss": 0.03646793263412741,
      "too_few": false
     }
    },
    "ece": 0.40049485774929267,
    "held_out": {
     "blunders": 0.1285512850420643,
     "kept": 0.015685113579938625,
     "no_cutoff": 150,
     "runs": 300
    },
    "n": 1664,
    "surest": {
     "cut": 0.9999990570378902,
     "error_rate": 14.257927138644915,
     "n": 416,
     "too_few": false
    },
    "too_few": false
   },
   "quality": {
    "auroc": 0.6606563485365958,
    "best_share": 0.5441256090958311,
    "blunder_share": 0.1824580400649702,
    "brier": 0.40111310849286447,
    "clusters": 94,
    "confidence_n": 1664,
    "confident_badly_wrong": 0.14362980769230768,
    "cost_coverage": 1.0,
    "decisions": 1847,
    "ece": 0.40049485774929267,
    "engine_scored": 1847,
    "error_hi": 25.309819313416035,
    "error_lo": 19.880791518808966,
    "error_rate": 22.47882081356059,
    "first_date": "2026-10-19",
    "group_sizes": [
     14,
     15,
     16,
     17,
     18,
     19,
     20,
     21,
     22,
     23,
     24,
     25,
     26
    ],
    "grouped": 183,
    "invalid": 0,
    "last_date": "2026-10-26",
    "mean_confidence": 0.9696054346723696,
    "over_budget": 0.016242555495397944,
    "own_cost": {
     "decisions": 1847,
     "estimate": true,
     "excluded_hours": 151.16666666666666,
     "gaps": [
      [
       "2026-10-19T20:30Z",
       "2026-10-19T20:41Z",
       11
      ],
      [
       "2026-10-19T20:45Z",
       "2026-10-19T22:27Z",
       102
      ],
      [
       "2026-10-20T02:51Z",
       "2026-10-20T04:53Z",
       122
      ],
      [
       "2026-10-20T05:06Z",
       "2026-10-20T06:01Z",
       55
      ],
      [
       "2026-10-20T06:16Z",
       "2026-10-21T01:09Z",
       1133
      ],
      [
       "2026-10-21T01:19Z",
       "2026-10-21T06:01Z",
       282
      ],
      [
       "2026-10-21T06:48Z",
       "2026-10-23T21:37Z",
       3768
      ],
      [
       "2026-10-23T21:41Z",
       "2026-10-25T07:55Z",
       2054
      ],
      [
       "2026-10-25T08:53Z",
       "2026-10-25T20:18Z",
       685
      ],
      [
       "2026-10-25T20:18Z",
       "2026-10-26T10:36Z",
       858
      ]
     ],
     "gpu_decisions": 1847,
     "gpu_usd": 15.17226129747234,
     "hosted_decisions": 0,
     "hosted_unpriced": 0,
     "hosted_usd": 0,
     "hours": 7.5490519444147655,
     "kind": "gpu",
     "parts": [
      {
       "gpus": 1,
       "hours": 1.5918261110782626,
       "machine": "dgx_spark",
       "usd": 0.33876897246444615,
       "usd_per_gpu_hour": 0.21281782608464228
      },
      {
       "gpus": 1,
       "hours": 5.957225833336503,
       "machine": "rtx_pro_6000",
       "usd": 14.833492325007894,
       "usd_per_gpu_hour": 2.49
      }
     ],
     "provenance": "content/own_cost.json: wall clock on our machines × hourly rate",
     "spark_rate": {
      "chf_per_h": 0.17465558152207,
      "electricity_chf_per_h": 0.02244858,
      "purchase_chf_per_h": 0.15220700152207,
      "usd_per_h": 0.21281782608464228
     },
     "usd": 15.17226129747234,
     "usd_per_1000": 8.214543203829097
    },
    "p95_loss": 0.2219608426094051,
    "probability_source": "logprobs",
    "retry_mean": 0.8970833784515431,
    "scored": 1847,
    "time_median": 0.311,
    "time_p90": 4.0316,
    "tokens_in": 4166.734163508392,
    "tokens_out": 1.2495939361126152,
    "too_few": false,
    "usd_per_1000": 8.214543203829097,
    "very_confident": 0.9074519230769231,
    "worst_mean": 0.3764484936128492
   },
   "risk": {
    "main": {
     "by_call": {
      "missed": 11.120296986476111,
      "needless": 2.057741849436654,
      "none": 2.5653740039178987,
      "right_risk": 1.546048672701387,
      "right_safety": 3.067688662488201,
      "wrong_risk": 2.121670638540337
     },
     "calls": {
      "missed": 341.0,
      "needless": 63.0,
      "right_risk": 194.0,
      "right_safety": 732.0,
      "wrong_risk": 49.0
     },
     "confidence": {
      "auroc_right_risk": 0.6012230857434068,
      "missed_mean_p": 0.97483588851821,
      "missed_n": 299,
      "taken_grouped": 45,
      "taken_n": 261,
      "taken_other": 0
     },
     "error_rate": 22.47882081356059,
     "found": {
      "den": 584.0,
      "n": 194.0,
      "share": 0.3321917808219178
     },
     "good": {
      "den": 306.0,
      "n": 194.0,
      "share": 0.6339869281045751
     },
     "judgement": 62.647324890152504,
     "judgement_hi": 65.03643951701261,
     "judgement_lo": 60.3838740457105,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.12564102564102564,
      "x": 0.6678082191780822,
      "y": 0.07924528301886792
     },
     "margin": 0.001,
     "missed": {
      "den": 584.0,
      "n": 341.0,
      "share": 0.583904109589041
     },
     "needless": {
      "den": 795.0,
      "n": 63.0,
      "share": 0.07924528301886792
     },
     "needless_by_loss": {
      "big": {
       "den": 795.0,
       "n": 39.0,
       "share": 0.04905660377358491
      },
      "medium": {
       "den": 795.0,
       "n": 22.0,
       "share": 0.027672955974842768
      },
      "small": {
       "den": 795.0,
       "n": 2.0,
       "share": 0.0025157232704402514
      }
     },
     "no_trade_off": 468.0,
     "reference": null,
     "right_safety": {
      "den": 795.0,
      "n": 732.0,
      "share": 0.9207547169811321
     },
     "risk_pays": 584.0,
     "safety_pays": 795.0,
     "scored": 1847,
     "sdt": {
      "aim": {
       "den": 243.0,
       "n": 194.0,
       "share": 0.7983539094650206
      },
      "beta": 1.3634606710699937,
      "caution": 0.8090589808989821,
      "caution_hi": 0.9013138555312011,
      "caution_lo": 0.7223429868305109,
      "cstar": 0.2594207355823052,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.1950705408797155,
      "dprime_hi": 1.356011648230725,
      "dprime_lo": 1.038255729140759,
      "false_alarm": {
       "den": 795.0,
       "n": 63.0,
       "share": 0.07924528301886792
      },
      "false_alarm_rate": 0.07977386934673367,
      "hit": {
       "den": 584.0,
       "n": 243.0,
       "share": 0.4160958904109589
      },
      "hit_rate": 0.41623931623931626,
      "missed_loss": 0.12046444887989077,
      "missed_n": 341.0,
      "na": null,
      "needless_loss": 0.12065553002887303,
      "needless_n": 63.0
     },
     "size": {
      "bad": {
       "mean": 7.607142857142857,
       "n": 112.0,
       "too_few": true
      },
      "right": {
       "mean": 10.989690721649485,
       "n": 194.0,
       "too_few": true
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 304.0,
        "n": 136.0,
        "share": 0.4473684210526316
       },
       "missed": {
        "den": 304.0,
        "n": 149.0,
        "share": 0.4901315789473684
       },
       "risk_pays": 304.0,
       "wrong": {
        "den": 304.0,
        "n": 19.0,
        "share": 0.0625
       }
      },
      "medium": {
       "found": {
        "den": 209.0,
        "n": 51.0,
        "share": 0.24401913875598086
       },
       "missed": {
        "den": 209.0,
        "n": 137.0,
        "share": 0.6555023923444976
       },
       "risk_pays": 209.0,
       "wrong": {
        "den": 209.0,
        "n": 21.0,
        "share": 0.10047846889952153
       }
      },
      "small": {
       "found": {
        "den": 71.0,
        "n": 7.0,
        "share": 0.09859154929577464
       },
       "missed": {
        "den": 71.0,
        "n": 55.0,
        "share": 0.7746478873239436
       },
       "risk_pays": 71.0,
       "wrong": {
        "den": 71.0,
        "n": 9.0,
        "share": 0.1267605633802817
       }
      }
     },
     "trade_off": 1379.0,
     "wrong": {
      "den": 584.0,
      "n": 49.0,
      "share": 0.0839041095890411
     }
    },
    "robust": {
     "by_call": {
      "missed": 10.677722027060401,
      "needless": 2.2718327459801064,
      "none": 2.5653740039178987,
      "right_risk": 1.4336932073364919,
      "right_safety": 3.5102636219039116,
      "wrong_risk": 2.01993520736178
     },
     "calls": {
      "missed": 286.0,
      "needless": 79.0,
      "right_risk": 182.0,
      "right_safety": 787.0,
      "wrong_risk": 45.0
     },
     "confidence": {
      "auroc_right_risk": 0.6308182263596276,
      "missed_mean_p": 0.9717690633413903,
      "missed_n": 253,
      "taken_grouped": 45,
      "taken_n": 261,
      "taken_other": 0
     },
     "error_rate": 22.47882081356059,
     "found": {
      "den": 513.0,
      "n": 182.0,
      "share": 0.35477582846003897
     },
     "good": {
      "den": 306.0,
      "n": 182.0,
      "share": 0.5947712418300654
     },
     "judgement": 63.17759049921442,
     "judgement_hi": 65.5889849786069,
     "judgement_lo": 60.970241564657904,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.13595166163141995,
      "x": 0.645224171539961,
      "y": 0.09122401847575058
     },
     "margin": 0.02,
     "missed": {
      "den": 513.0,
      "n": 286.0,
      "share": 0.557504873294347
     },
     "needless": {
      "den": 866.0,
      "n": 79.0,
      "share": 0.09122401847575058
     },
     "needless_by_loss": {
      "big": {
       "den": 866.0,
       "n": 43.0,
       "share": 0.049653579676674366
      },
      "medium": {
       "den": 866.0,
       "n": 25.0,
       "share": 0.028868360277136258
      },
      "small": {
       "den": 866.0,
       "n": 11.0,
       "share": 0.012702078521939953
      }
     },
     "no_trade_off": 468.0,
     "reference": null,
     "right_safety": {
      "den": 866.0,
      "n": 787.0,
      "share": 0.9087759815242494
     },
     "risk_pays": 513.0,
     "safety_pays": 866.0,
     "scored": 1847,
     "sdt": {
      "aim": {
       "den": 227.0,
       "n": 182.0,
       "share": 0.801762114537445
      },
      "beta": 1.3002809898277086,
      "caution": 0.7373745733239244,
      "caution_hi": 0.818024862474868,
      "caution_lo": 0.6613211310008141,
      "cstar": 0.22139558535868598,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.1860235907920635,
      "dprime_hi": 1.3531028315959266,
      "dprime_lo": 1.0282277665095578,
      "false_alarm": {
       "den": 866.0,
       "n": 79.0,
       "share": 0.09122401847575058
      },
      "false_alarm_rate": 0.09169550173010381,
      "hit": {
       "den": 513.0,
       "n": 227.0,
       "share": 0.442495126705653
      },
      "hit_rate": 0.44260700389105057,
      "missed_loss": 0.1379143537341298,
      "missed_n": 286.0,
      "na": null,
      "needless_loss": 0.10622974890696851,
      "needless_n": 79.0
     },
     "size": {
      "bad": {
       "mean": 8.298387096774194,
       "n": 124.0,
       "too_few": true
      },
      "right": {
       "mean": 10.741758241758241,
       "n": 182.0,
       "too_few": true
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 304.0,
        "n": 133.0,
        "share": 0.4375
       },
       "missed": {
        "den": 304.0,
        "n": 149.0,
        "share": 0.4901315789473684
       },
       "risk_pays": 304.0,
       "wrong": {
        "den": 304.0,
        "n": 22.0,
        "share": 0.07236842105263158
       }
      },
      "medium": {
       "found": {
        "den": 209.0,
        "n": 49.0,
        "share": 0.23444976076555024
       },
       "missed": {
        "den": 209.0,
        "n": 137.0,
        "share": 0.6555023923444976
       },
       "risk_pays": 209.0,
       "wrong": {
        "den": 209.0,
        "n": 23.0,
        "share": 0.11004784688995216
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 1379.0,
     "wrong": {
      "den": 513.0,
      "n": 45.0,
      "share": 0.08771929824561403
     }
    }
   },
   "slug": "gemma-4-31b-off"
  },
  "Gemma 4 31B (on)": {
   "close": {
    "close": {
     "den": 169,
     "n": 0,
     "share": 0.0,
     "small": true
    },
    "decisions": 2122,
    "definition": "A decision is a close choice when the second most likely play has at least half the probability of the first (internally: split).",
    "eligible": 169,
    "exclusions": {
     "grouped": 175,
     "not_audited": 599,
     "sparse": 1179
    },
    "groups": {
     "close": {
      "by_count": [
       0,
       0,
       0,
       0,
       0
      ],
      "differ": {
       "hitting": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       },
       "points": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       },
       "prime": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       },
       "race": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       }
      },
      "gap": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "loss1": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "loss2": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "n": 0,
      "p1": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "p2": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "psum": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "psum_bins": [
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0
      ]
     },
     "other": {
      "by_count": [
       53,
       73,
       39,
       4,
       0
      ],
      "differ": {
       "hitting": {
        "den": 169,
        "n": 68,
        "share": 0.40236686390532544,
        "small": true
       },
       "points": {
        "den": 169,
        "n": 18,
        "share": 0.10650887573964497,
        "small": true
       },
       "prime": {
        "den": 169,
        "n": 48,
        "share": 0.28402366863905326,
        "small": true
       },
       "race": {
        "den": 169,
        "n": 29,
        "share": 0.17159763313609466,
        "small": true
       }
      },
      "gap": {
       "mean": 0.14192516528643095,
       "median": 0.05368685722351074,
       "n": 169
      },
      "loss1": {
       "mean": 0.008113073526754886,
       "median": 0.0,
       "n": 169
      },
      "loss2": {
       "mean": 0.13727601886500973,
       "median": 0.04500126838684082,
       "n": 169
      },
      "n": 169,
      "p1": {
       "mean": 0.9999734228884493,
       "median": 0.9999999006891412,
       "n": 169
      },
      "p2": {
       "mean": 2.6537929015719625e-05,
       "median": 9.32948448314427e-08,
       "n": 169
      },
      "psum": {
       "mean": 0.9999999608174651,
       "median": 1.0,
       "n": 169
      },
      "psum_bins": [
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       169
      ]
     }
    },
    "label": "Probabilities over plays, not sampled reasoning.",
    "margins": {
     "main": {
      "classes": {
       "none": {
        "close": {
         "den": 98,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 98,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       },
       "risk": {
        "close": {
         "den": 14,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 14,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       },
       "safety": {
        "close": {
         "den": 57,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 57,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       }
      },
      "margin": 0.001,
      "missed": {
       "close": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       },
       "group": 13,
       "na": "N/A: no close choices in this group",
       "other": {
        "den": 13,
        "n": 10,
        "share": 0.7692307692307693,
        "small": true
       }
      },
      "stake": {
       "big": {
        "den": 2,
        "n": 0,
        "share": 0.0,
        "small": true
       },
       "medium": {
        "den": 8,
        "n": 0,
        "share": 0.0,
        "small": true
       },
       "small": {
        "den": 4,
        "n": 0,
        "share": 0.0,
        "small": true
       }
      }
     },
     "robust": {
      "classes": {
       "none": {
        "close": {
         "den": 98,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 98,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       },
       "risk": {
        "close": {
         "den": 10,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 10,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       },
       "safety": {
        "close": {
         "den": 61,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 61,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       }
      },
      "margin": 0.02,
      "missed": {
       "close": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       },
       "group": 9,
       "na": "N/A: no close choices in this group",
       "other": {
        "den": 9,
        "n": 7,
        "share": 0.7777777777777778,
        "small": true
       }
      },
      "stake": {
       "big": {
        "den": 2,
        "n": 0,
        "share": 0.0,
        "small": true
       },
       "medium": {
        "den": 8,
        "n": 0,
        "share": 0.0,
        "small": true
       },
       "small": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       }
      }
     }
    },
    "na": null,
    "risk_close": {
     "den": 0,
     "n": 0,
     "share": null,
     "small": false
    },
    "risk_close_of_eligible": {
     "den": 169,
     "n": 0,
     "share": 0.0,
     "small": true
    },
    "risk_pair_other": {
     "den": 169,
     "n": 68,
     "share": 0.40236686390532544,
     "small": true
    },
    "ties": {
     "close": 0,
     "member": 0,
     "order": 0,
     "positive": 0,
     "top_two": 0,
     "zero": 0
    }
   },
   "confidence": {
    "auroc_best": 0.6411258919618781,
    "auroc_blunder": 0.5648871242945268,
    "confident_badly_wrong": {
     "den": 1870,
     "n": 265,
     "share": 0.14171122994652408
    },
    "cutoffs": {
     "0.9": {
      "blunders": {
       "den": 1866,
       "n": 265,
       "share": 0.1420150053590568
      },
      "kept": {
       "den": 1870,
       "n": 1866,
       "share": 0.9978609625668449
      },
      "mean_loss": 0.03640420548018918,
      "too_few": false
     },
     "0.99": {
      "blunders": {
       "den": 1856,
       "n": 264,
       "share": 0.14224137931034483
      },
      "kept": {
       "den": 1870,
       "n": 1856,
       "share": 0.9925133689839573
      },
      "mean_loss": 0.03618094684748814,
      "too_few": false
     }
    },
    "ece": 0.3764133583922834,
    "held_out": {
     "blunders": 0.10989010989010989,
     "kept": 0.09381443298969072,
     "no_cutoff": 299,
     "runs": 300
    },
    "n": 1870,
    "surest": {
     "cut": 0.9999999716149833,
     "error_rate": 14.009219809220387,
     "n": 468,
     "too_few": false
    },
    "too_few": false
   },
   "quality": {
    "auroc": 0.6411258919618781,
    "best_share": 0.5914231856738925,
    "blunder_share": 0.15598491988689916,
    "brier": 0.37584372483462875,
    "clusters": 104,
    "confidence_n": 1870,
    "confident_badly_wrong": 0.14171122994652408,
    "cost_coverage": 1.0,
    "decisions": 2122,
    "ece": 0.3764133583922834,
    "engine_scored": 2122,
    "error_hi": 21.731960517592608,
    "error_lo": 17.47346753644494,
    "error_rate": 19.575135612072078,
    "first_date": "2026-10-19",
    "group_sizes": [
     14,
     15,
     16,
     17,
     18,
     19,
     20,
     21,
     22,
     23,
     24,
     25,
     26
    ],
    "grouped": 252,
    "invalid": 0,
    "last_date": "2026-10-26",
    "mean_confidence": 0.9988732514404117,
    "over_budget": 0.9637134778510839,
    "own_cost": {
     "decisions": 2122,
     "estimate": true,
     "excluded_hours": 134.01666666666668,
     "gaps": [
      [
       "2026-10-19T20:05Z",
       "2026-10-19T20:16Z",
       11
      ],
      [
       "2026-10-19T21:11Z",
       "2026-10-19T21:31Z",
       19
      ],
      [
       "2026-10-19T21:31Z",
       "2026-10-19T21:48Z",
       18
      ],
      [
       "2026-10-19T21:48Z",
       "2026-10-19T22:28Z",
       40
      ],
      [
       "2026-10-20T03:15Z",
       "2026-10-20T03:28Z",
       13
      ],
      [
       "2026-10-20T04:23Z",
       "2026-10-20T04:37Z",
       14
      ],
      [
       "2026-10-20T04:37Z",
       "2026-10-20T04:48Z",
       11
      ],
      [
       "2026-10-20T07:29Z",
       "2026-10-20T08:00Z",
       30
      ],
      [
       "2026-10-20T08:11Z",
       "2026-10-20T08:47Z",
       36
      ],
      [
       "2026-10-20T08:51Z",
       "2026-10-20T09:52Z",
       61
      ],
      [
       "2026-10-20T10:43Z",
       "2026-10-20T10:53Z",
       10
      ],
      [
       "2026-10-20T10:59Z",
       "2026-10-20T11:56Z",
       57
      ],
      [
       "2026-10-20T12:03Z",
       "2026-10-20T20:15Z",
       493
      ],
      [
       "2026-10-20T20:22Z",
       "2026-10-20T20:35Z",
       13
      ],
      [
       "2026-10-20T20:35Z",
       "2026-10-20T20:49Z",
       14
      ],
      [
       "2026-10-20T21:07Z",
       "2026-10-20T21:26Z",
       19
      ],
      [
       "2026-10-20T21:26Z",
       "2026-10-20T22:04Z",
       38
      ],
      [
       "2026-10-20T22:04Z",
       "2026-10-20T22:15Z",
       12
      ],
      [
       "2026-10-20T22:15Z",
       "2026-10-20T22:27Z",
       12
      ],
      [
       "2026-10-20T22:27Z",
       "2026-10-20T22:45Z",
       17
      ],
      [
       "2026-10-20T22:45Z",
       "2026-10-20T23:08Z",
       23
      ],
      [
       "2026-10-20T23:08Z",
       "2026-10-21T00:18Z",
       70
      ],
      [
       "2026-10-21T00:18Z",
       "2026-10-21T01:16Z",
       58
      ],
      [
       "2026-10-21T01:16Z",
       "2026-10-21T01:26Z",
       10
      ],
      [
       "2026-10-21T02:03Z",
       "2026-10-21T06:18Z",
       254
      ],
      [
       "2026-10-21T06:27Z",
       "2026-10-21T06:44Z",
       17
      ],
      [
       "2026-10-21T06:44Z",
       "2026-10-21T07:05Z",
       21
      ],
      [
       "2026-10-21T07:05Z",
       "2026-10-21T07:15Z",
       10
      ],
      [
       "2026-10-21T07:15Z",
       "2026-10-21T10:53Z",
       218
      ],
      [
       "2026-10-21T10:53Z",
       "2026-10-21T11:07Z",
       14
      ],
      [
       "2026-10-21T11:07Z",
       "2026-10-21T11:28Z",
       21
      ],
      [
       "2026-10-21T11:28Z",
       "2026-10-21T11:49Z",
       21
      ],
      [
       "2026-10-21T11:49Z",
       "2026-10-21T12:06Z",
       17
      ],
      [
       "2026-10-21T12:06Z",
       "2026-10-21T12:32Z",
       26
      ],
      [
       "2026-10-21T12:32Z",
       "2026-10-21T23:36Z",
       664
      ],
      [
       "2026-10-21T23:36Z",
       "2026-10-21T23:55Z",
       19
      ],
      [
       "2026-10-21T23:55Z",
       "2026-10-22T00:46Z",
       52
      ],
      [
       "2026-10-22T00:46Z",
       "2026-10-22T01:07Z",
       21
      ],
      [
       "2026-10-22T01:07Z",
       "2026-10-22T01:32Z",
       25
      ],
      [
       "2026-10-22T01:32Z",
       "2026-10-22T01:50Z",
       17
      ],
      [
       "2026-10-22T01:56Z",
       "2026-10-22T02:12Z",
       16
      ],
      [
       "2026-10-22T02:12Z",
       "2026-10-22T02:35Z",
       23
      ],
      [
       "2026-10-22T02:35Z",
       "2026-10-22T02:53Z",
       18
      ],
      [
       "2026-10-22T02:53Z",
       "2026-10-22T03:52Z",
       59
      ],
      [
       "2026-10-22T03:52Z",
       "2026-10-22T04:17Z",
       25
      ],
      [
       "2026-10-22T04:17Z",
       "2026-10-22T05:42Z",
       85
      ],
      [
       "2026-10-22T05:42Z",
       "2026-10-22T15:03Z",
       561
      ],
      [
       "2026-10-22T15:03Z",
       "2026-10-22T15:40Z",
       37
      ],
      [
       "2026-10-22T16:19Z",
       "2026-10-22T23:27Z",
       428
      ],
      [
       "2026-10-23T01:30Z",
       "2026-10-23T01:47Z",
       17
      ],
      [
       "2026-10-23T02:50Z",
       "2026-10-23T21:37Z",
       1127
      ],
      [
       "2026-10-23T22:35Z",
       "2026-10-24T06:49Z",
       494
      ],
      [
       "2026-10-24T08:12Z",
       "2026-10-25T07:56Z",
       1424
      ],
      [
       "2026-10-25T11:44Z",
       "2026-10-25T11:58Z",
       14
      ],
      [
       "2026-10-25T12:17Z",
       "2026-10-25T13:02Z",
       46
      ],
      [
       "2026-10-25T13:38Z",
       "2026-10-25T20:18Z",
       400
      ],
      [
       "2026-10-25T20:46Z",
       "2026-10-25T21:56Z",
       70
      ],
      [
       "2026-10-25T23:18Z",
       "2026-10-25T23:43Z",
       25
      ],
      [
       "2026-10-25T23:43Z",
       "2026-10-25T23:56Z",
       14
      ],
      [
       "2026-10-26T00:21Z",
       "2026-10-26T10:37Z",
       616
      ],
      [
       "2026-10-26T10:41Z",
       "2026-10-26T10:51Z",
       10
      ],
      [
       "2026-10-26T11:08Z",
       "2026-10-26T11:24Z",
       16
      ]
     ],
     "gpu_decisions": 2122,
     "gpu_usd": 74.14828364608573,
     "hosted_decisions": 0,
     "hosted_unpriced": 0,
     "hosted_usd": 0,
     "hours": 27.19192055556512,
     "kind": "gpu",
     "parts": [
      {
       "gpus": 1,
       "hours": 1.287808611061838,
       "machine": "dgx_spark",
       "usd": 0.274068629019263,
       "usd_per_gpu_hour": 0.21281782608464228
      },
      {
       "gpus": 1,
       "hours": 22.139864444401557,
       "machine": "rtx_pro_6000",
       "usd": 55.12826246655988,
       "usd_per_gpu_hour": 2.49
      },
      {
       "gpus": 2,
       "hours": 3.764247500101725,
       "machine": "rtx_pro_6000",
       "usd": 18.745952550506594,
       "usd_per_gpu_hour": 2.49
      }
     ],
     "provenance": "content/own_cost.json: wall clock on our machines × hourly rate",
     "spark_rate": {
      "chf_per_h": 0.17465558152207,
      "electricity_chf_per_h": 0.02244858,
      "purchase_chf_per_h": 0.15220700152207,
      "usd_per_h": 0.21281782608464228
     },
     "usd": 74.14828364608573,
     "usd_per_1000": 34.942640738023435
    },
    "p95_loss": 0.20325340330600725,
    "probability_source": "logprobs",
    "retry_mean": 15.291248821866164,
    "scored": 2122,
    "time_median": 120.5985,
    "time_p90": 394.2825,
    "tokens_in": 14745.608388312912,
    "tokens_out": 5598.933081998115,
    "too_few": false,
    "usd_per_1000": 34.942640738023435,
    "very_confident": 0.9978609625668449,
    "worst_mean": 0.3483544635885167
   },
   "risk": {
    "main": {
     "by_call": {
      "missed": 12.790185861942568,
      "needless": 0.5388124639644137,
      "none": 1.0812876782925145,
      "right_risk": 0.9583633508915951,
      "right_safety": 2.9969350992228825,
      "wrong_risk": 1.2095511577581033
     },
     "calls": {
      "missed": 454.0,
      "needless": 26.0,
      "right_risk": 223.0,
      "right_safety": 878.0,
      "wrong_risk": 23.0
     },
     "confidence": {
      "auroc_right_risk": 0.5187837837837838,
      "missed_mean_p": 0.9972724127351217,
      "missed_n": 385,
      "taken_grouped": 47,
      "taken_n": 225,
      "taken_other": 0
     },
     "error_rate": 19.575135612072078,
     "found": {
      "den": 700.0,
      "n": 223.0,
      "share": 0.31857142857142856
     },
     "good": {
      "den": 272.0,
      "n": 223.0,
      "share": 0.8198529411764706
     },
     "judgement": 64.4905183312263,
     "judgement_hi": 66.20745194260151,
     "judgement_lo": 62.8537270567075,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.04821802935010482,
      "x": 0.6814285714285714,
      "y": 0.028761061946902654
     },
     "margin": 0.001,
     "missed": {
      "den": 700.0,
      "n": 454.0,
      "share": 0.6485714285714286
     },
     "needless": {
      "den": 904.0,
      "n": 26.0,
      "share": 0.028761061946902654
     },
     "needless_by_loss": {
      "big": {
       "den": 904.0,
       "n": 11.0,
       "share": 0.012168141592920354
      },
      "medium": {
       "den": 904.0,
       "n": 10.0,
       "share": 0.011061946902654867
      },
      "small": {
       "den": 904.0,
       "n": 5.0,
       "share": 0.0055309734513274336
      }
     },
     "no_trade_off": 518.0,
     "reference": null,
     "right_safety": {
      "den": 904.0,
      "n": 878.0,
      "share": 0.9712389380530974
     },
     "risk_pays": 700.0,
     "safety_pays": 904.0,
     "scored": 2122,
     "sdt": {
      "aim": {
       "den": 246.0,
       "n": 223.0,
       "share": 0.9065040650406504
      },
      "beta": 0.9499782742824628,
      "caution": 1.136175453849757,
      "caution_hi": 1.2327024961843753,
      "caution_lo": 1.05385644658735,
      "cstar": -0.03397160351794271,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.5105605420805492,
      "dprime_hi": 1.6875083662190107,
      "dprime_lo": 1.3588519741533376,
      "false_alarm": {
       "den": 904.0,
       "n": 26.0,
       "share": 0.028761061946902654
      },
      "false_alarm_rate": 0.029281767955801105,
      "hit": {
       "den": 700.0,
       "n": 246.0,
       "share": 0.3514285714285714
      },
      "hit_rate": 0.3516405135520685,
      "missed_loss": 0.11956288281516357,
      "missed_n": 454.0,
      "na": null,
      "needless_loss": 0.08795077296403739,
      "needless_n": 26.0
     },
     "size": {
      "bad": {
       "mean": 8.224489795918368,
       "n": 49.0,
       "too_few": true
      },
      "right": {
       "mean": 10.327354260089686,
       "n": 223.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 375.0,
        "n": 164.0,
        "share": 0.43733333333333335
       },
       "missed": {
        "den": 375.0,
        "n": 199.0,
        "share": 0.5306666666666666
       },
       "risk_pays": 375.0,
       "wrong": {
        "den": 375.0,
        "n": 12.0,
        "share": 0.032
       }
      },
      "medium": {
       "found": {
        "den": 238.0,
        "n": 55.0,
        "share": 0.23109243697478993
       },
       "missed": {
        "den": 238.0,
        "n": 177.0,
        "share": 0.7436974789915967
       },
       "risk_pays": 238.0,
       "wrong": {
        "den": 238.0,
        "n": 6.0,
        "share": 0.025210084033613446
       }
      },
      "small": {
       "found": {
        "den": 87.0,
        "n": 4.0,
        "share": 0.04597701149425287
       },
       "missed": {
        "den": 87.0,
        "n": 78.0,
        "share": 0.896551724137931
       },
       "risk_pays": 87.0,
       "wrong": {
        "den": 87.0,
        "n": 5.0,
        "share": 0.05747126436781609
       }
      }
     },
     "trade_off": 1604.0,
     "wrong": {
      "den": 700.0,
      "n": 23.0,
      "share": 0.032857142857142856
     }
    },
    "robust": {
     "by_call": {
      "missed": 12.280990846251452,
      "needless": 0.782809849621326,
      "none": 1.0812876782925145,
      "right_risk": 0.9077945539796274,
      "right_safety": 3.506130114913998,
      "wrong_risk": 1.0161225690131588
     },
     "calls": {
      "missed": 376.0,
      "needless": 35.0,
      "right_risk": 213.0,
      "right_safety": 956.0,
      "wrong_risk": 24.0
     },
     "confidence": {
      "auroc_right_risk": 0.5192486085343229,
      "missed_mean_p": 0.9972158234779105,
      "missed_n": 321,
      "taken_grouped": 47,
      "taken_n": 225,
      "taken_other": 0
     },
     "error_rate": 19.575135612072078,
     "found": {
      "den": 613.0,
      "n": 213.0,
      "share": 0.3474714518760196
     },
     "good": {
      "den": 272.0,
      "n": 213.0,
      "share": 0.7830882352941176
     },
     "judgement": 65.60767955646496,
     "judgement_hi": 67.54887619692241,
     "judgement_lo": 63.76867483011015,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.06,
      "x": 0.6525285481239804,
      "y": 0.035317860746720484
     },
     "margin": 0.02,
     "missed": {
      "den": 613.0,
      "n": 376.0,
      "share": 0.6133768352365416
     },
     "needless": {
      "den": 991.0,
      "n": 35.0,
      "share": 0.035317860746720484
     },
     "needless_by_loss": {
      "big": {
       "den": 991.0,
       "n": 16.0,
       "share": 0.016145307769929364
      },
      "medium": {
       "den": 991.0,
       "n": 10.0,
       "share": 0.010090817356205853
      },
      "small": {
       "den": 991.0,
       "n": 9.0,
       "share": 0.009081735620585268
      }
     },
     "no_trade_off": 518.0,
     "reference": null,
     "right_safety": {
      "den": 991.0,
      "n": 956.0,
      "share": 0.9646821392532795
     },
     "risk_pays": 613.0,
     "safety_pays": 991.0,
     "scored": 2122,
     "sdt": {
      "aim": {
       "den": 237.0,
       "n": 213.0,
       "share": 0.8987341772151899
      },
      "beta": 1.1070209870486556,
      "caution": 1.044738019303215,
      "caution_hi": 1.1421944881247905,
      "caution_lo": 0.9578361459263605,
      "cstar": 0.06714704211095375,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.5141785675233457,
      "dprime_hi": 1.6837314141066382,
      "dprime_lo": 1.3534044210934155,
      "false_alarm": {
       "den": 991.0,
       "n": 35.0,
       "share": 0.035317860746720484
      },
      "false_alarm_rate": 0.03578629032258065,
      "hit": {
       "den": 613.0,
       "n": 237.0,
       "share": 0.3866231647634584
      },
      "hit_rate": 0.38680781758957655,
      "missed_loss": 0.13861841795609353,
      "missed_n": 376.0,
      "na": null,
      "needless_loss": 0.09492128576551165,
      "needless_n": 35.0
     },
     "size": {
      "bad": {
       "mean": 9.508474576271187,
       "n": 59.0,
       "too_few": true
      },
      "right": {
       "mean": 10.070422535211268,
       "n": 213.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 375.0,
        "n": 163.0,
        "share": 0.43466666666666665
       },
       "missed": {
        "den": 375.0,
        "n": 199.0,
        "share": 0.5306666666666666
       },
       "risk_pays": 375.0,
       "wrong": {
        "den": 375.0,
        "n": 13.0,
        "share": 0.034666666666666665
       }
      },
      "medium": {
       "found": {
        "den": 238.0,
        "n": 50.0,
        "share": 0.21008403361344538
       },
       "missed": {
        "den": 238.0,
        "n": 177.0,
        "share": 0.7436974789915967
       },
       "risk_pays": 238.0,
       "wrong": {
        "den": 238.0,
        "n": 11.0,
        "share": 0.046218487394957986
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 1604.0,
     "wrong": {
      "den": 613.0,
      "n": 24.0,
      "share": 0.03915171288743882
     }
    }
   },
   "slug": "gemma-4-31b-on"
  },
  "Grok 4.7 (low)": {
   "close": {
    "decisions": 2152,
    "definition": "A decision is a close choice when the second most likely play has at least half the probability of the first (internally: split).",
    "eligible": 0,
    "exclusions": {
     "no_probabilities": 1741,
     "not_audited": 411
    },
    "label": "Probabilities over plays, not sampled reasoning.",
    "na": "N/A: no full probability distribution"
   },
   "confidence": null,
   "quality": {
    "best_share": 0.6078066914498141,
    "blunder_share": 0.15102230483271376,
    "clusters": 102,
    "cost_coverage": 1.0,
    "decisions": 2152,
    "engine_scored": 2152,
    "error_hi": 21.404774317251597,
    "error_lo": 16.943950338913243,
    "error_rate": 19.128085505286556,
    "first_date": "2026-10-20",
    "group_sizes": [
     14,
     15,
     16,
     17,
     18,
     19,
     20,
     21,
     22,
     23,
     24,
     25,
     26
    ],
    "grouped": 250,
    "invalid": 0,
    "last_date": "2026-10-26",
    "over_budget": 0.4270446096654275,
    "p95_loss": 0.20881198644638047,
    "probability_source": "choice_only",
    "retry_mean": 14.042878717472119,
    "scored": 2152,
    "time_median": 24.317999999999998,
    "time_p90": 110.19290000000004,
    "tokens_in": 5999.413104089219,
    "tokens_out": 3468.961431226766,
    "too_few": false,
    "usd_per_1000": 32.812594795539034,
    "worst_mean": 0.35378020836247337
   },
   "risk": {
    "main": {
     "by_call": {
      "missed": 10.404514675080554,
      "needless": 1.9887585026734824,
      "none": 0.5103734124548817,
      "right_risk": 1.9002424684598986,
      "right_safety": 2.040249278221875,
      "wrong_risk": 2.2839471683958648
     },
     "calls": {
      "missed": 348.0,
      "needless": 64.0,
      "right_risk": 295.0,
      "right_safety": 852.0,
      "wrong_risk": 54.0
     },
     "confidence": null,
     "error_rate": 19.128085505286556,
     "found": {
      "den": 697.0,
      "n": 295.0,
      "share": 0.4232424677187948
     },
     "good": {
      "den": 413.0,
      "n": 295.0,
      "share": 0.7142857142857143
     },
     "judgement": 67.66867360428034,
     "judgement_hi": 69.493489041885,
     "judgement_lo": 65.95152290971117,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.13432835820895522,
      "x": 0.5767575322812052,
      "y": 0.06986899563318777
     },
     "margin": 0.001,
     "missed": {
      "den": 697.0,
      "n": 348.0,
      "share": 0.49928263988522237
     },
     "needless": {
      "den": 916.0,
      "n": 64.0,
      "share": 0.06986899563318777
     },
     "needless_by_loss": {
      "big": {
       "den": 916.0,
       "n": 31.0,
       "share": 0.03384279475982533
      },
      "medium": {
       "den": 916.0,
       "n": 19.0,
       "share": 0.02074235807860262
      },
      "small": {
       "den": 916.0,
       "n": 14.0,
       "share": 0.015283842794759825
      }
     },
     "no_trade_off": 539.0,
     "reference": null,
     "right_safety": {
      "den": 916.0,
      "n": 852.0,
      "share": 0.9301310043668122
     },
     "risk_pays": 697.0,
     "safety_pays": 916.0,
     "scored": 2152,
     "sdt": {
      "aim": {
       "den": 349.0,
       "n": 295.0,
       "share": 0.8452722063037249
      },
      "beta": 1.365910324909029,
      "caution": 0.7357411591070739,
      "caution_hi": 0.8141393871534625,
      "caution_lo": 0.6665197037640483,
      "cstar": 0.21139361236789453,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.4750734781304389,
      "dprime_hi": 1.6320125822195823,
      "dprime_lo": 1.3358724601526142,
      "false_alarm": {
       "den": 916.0,
       "n": 64.0,
       "share": 0.06986899563318777
      },
      "false_alarm_rate": 0.0703380588876772,
      "hit": {
       "den": 697.0,
       "n": 349.0,
       "share": 0.5007173601147776
      },
      "hit_rate": 0.5007163323782235,
      "missed_loss": 0.12868112402743306,
      "missed_n": 348.0,
      "na": null,
      "needless_loss": 0.1337440093047917,
      "needless_n": 64.0
     },
     "size": {
      "bad": {
       "mean": 9.220338983050848,
       "n": 118.0,
       "too_few": true
      },
      "right": {
       "mean": 9.315254237288135,
       "n": 295.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 391.0,
        "n": 201.0,
        "share": 0.5140664961636828
       },
       "missed": {
        "den": 391.0,
        "n": 169.0,
        "share": 0.4322250639386189
       },
       "risk_pays": 391.0,
       "wrong": {
        "den": 391.0,
        "n": 21.0,
        "share": 0.05370843989769821
       }
      },
      "medium": {
       "found": {
        "den": 223.0,
        "n": 76.0,
        "share": 0.34080717488789236
       },
       "missed": {
        "den": 223.0,
        "n": 121.0,
        "share": 0.5426008968609866
       },
       "risk_pays": 223.0,
       "wrong": {
        "den": 223.0,
        "n": 26.0,
        "share": 0.11659192825112108
       }
      },
      "small": {
       "found": {
        "den": 83.0,
        "n": 18.0,
        "share": 0.21686746987951808
       },
       "missed": {
        "den": 83.0,
        "n": 58.0,
        "share": 0.6987951807228916
       },
       "risk_pays": 83.0,
       "wrong": {
        "den": 83.0,
        "n": 7.0,
        "share": 0.08433734939759036
       }
      }
     },
     "trade_off": 1613.0,
     "wrong": {
      "den": 697.0,
      "n": 54.0,
      "share": 0.07747489239598278
     }
    },
    "robust": {
     "by_call": {
      "missed": 10.123042722118388,
      "needless": 2.150829389802142,
      "none": 0.5103734124548817,
      "right_risk": 1.656552409704733,
      "right_safety": 2.3217212311840414,
      "wrong_risk": 2.3655663400223705
     },
     "calls": {
      "missed": 290.0,
      "needless": 89.0,
      "right_risk": 263.0,
      "right_safety": 910.0,
      "wrong_risk": 61.0
     },
     "confidence": null,
     "error_rate": 19.128085505286556,
     "found": {
      "den": 614.0,
      "n": 263.0,
      "share": 0.42833876221498374
     },
     "good": {
      "den": 413.0,
      "n": 263.0,
      "share": 0.6368038740920097
     },
     "judgement": 66.96248365629474,
     "judgement_hi": 69.05791346595883,
     "judgement_lo": 64.93571206363312,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.1737891737891738,
      "x": 0.5716612377850163,
      "y": 0.0890890890890891
     },
     "margin": 0.02,
     "missed": {
      "den": 614.0,
      "n": 290.0,
      "share": 0.4723127035830619
     },
     "needless": {
      "den": 999.0,
      "n": 89.0,
      "share": 0.0890890890890891
     },
     "needless_by_loss": {
      "big": {
       "den": 999.0,
       "n": 32.0,
       "share": 0.03203203203203203
      },
      "medium": {
       "den": 999.0,
       "n": 24.0,
       "share": 0.024024024024024024
      },
      "small": {
       "den": 999.0,
       "n": 33.0,
       "share": 0.03303303303303303
      }
     },
     "no_trade_off": 539.0,
     "reference": null,
     "right_safety": {
      "den": 999.0,
      "n": 910.0,
      "share": 0.9109109109109109
     },
     "risk_pays": 614.0,
     "safety_pays": 999.0,
     "scored": 2152,
     "sdt": {
      "aim": {
       "den": 324.0,
       "n": 263.0,
       "share": 0.8117283950617284
      },
      "beta": 1.1264190868624469,
      "caution": 0.6372479798659383,
      "caution_hi": 0.7088614048305211,
      "caution_lo": 0.5727217382036486,
      "cstar": 0.08423784825557437,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.41318485403055,
      "dprime_hi": 1.566258094924094,
      "dprime_lo": 1.2648874594714536,
      "false_alarm": {
       "den": 999.0,
       "n": 89.0,
       "share": 0.0890890890890891
      },
      "false_alarm_rate": 0.0895,
      "hit": {
       "den": 614.0,
       "n": 324.0,
       "share": 0.5276872964169381
      },
      "hit_rate": 0.5276422764227642,
      "missed_loss": 0.15023991681378462,
      "missed_n": 290.0,
      "na": null,
      "needless_loss": 0.1040131426259373,
      "needless_n": 89.0
     },
     "size": {
      "bad": {
       "mean": 10.046666666666667,
       "n": 150.0,
       "too_few": true
      },
      "right": {
       "mean": 8.855513307984792,
       "n": 263.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 391.0,
        "n": 197.0,
        "share": 0.5038363171355499
       },
       "missed": {
        "den": 391.0,
        "n": 169.0,
        "share": 0.4322250639386189
       },
       "risk_pays": 391.0,
       "wrong": {
        "den": 391.0,
        "n": 25.0,
        "share": 0.0639386189258312
       }
      },
      "medium": {
       "found": {
        "den": 223.0,
        "n": 66.0,
        "share": 0.29596412556053814
       },
       "missed": {
        "den": 223.0,
        "n": 121.0,
        "share": 0.5426008968609866
       },
       "risk_pays": 223.0,
       "wrong": {
        "den": 223.0,
        "n": 36.0,
        "share": 0.16143497757847533
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 1613.0,
     "wrong": {
      "den": 614.0,
      "n": 61.0,
      "share": 0.0993485342019544
     }
    }
   },
   "slug": "grok-4-7-off"
  },
  "Jev 1.14.0": {
   "close": {
    "close": {
     "den": 507,
     "n": 231,
     "share": 0.4556213017751479,
     "small": false
    },
    "decisions": 1912,
    "definition": "A decision is a close choice when the second most likely play has at least half the probability of the first (internally: split).",
    "eligible": 507,
    "exclusions": {
     "not_audited": 1405
    },
    "groups": {
     "close": {
      "by_count": [
       70,
       78,
       59,
       22,
       2
      ],
      "differ": {
       "hitting": {
        "den": 231,
        "n": 107,
        "share": 0.46320346320346323,
        "small": false
       },
       "points": {
        "den": 231,
        "n": 73,
        "share": 0.31601731601731603,
        "small": false
       },
       "prime": {
        "den": 231,
        "n": 58,
        "share": 0.2510822510822511,
        "small": false
       },
       "race": {
        "den": 231,
        "n": 32,
        "share": 0.13852813852813853,
        "small": false
       }
      },
      "gap": {
       "mean": 0.09594615803652512,
       "median": 0.058322906494140625,
       "n": 231
      },
      "loss1": {
       "mean": 0.07696300306361475,
       "median": 0.03133344650268555,
       "n": 231
      },
      "loss2": {
       "mean": 0.09723944671742328,
       "median": 0.05385744571685791,
       "n": 231
      },
      "n": 231,
      "p1": {
       "mean": 0.3769154750972933,
       "median": 0.36,
       "n": 231
      },
      "p2": {
       "mean": 0.2699449035812672,
       "median": 0.27,
       "n": 231
      },
      "psum": {
       "mean": 0.6468603786785605,
       "median": 0.63,
       "n": 231
      },
      "psum_bins": [
       0,
       0,
       4,
       13,
       32,
       46,
       56,
       29,
       25,
       26
      ]
     },
     "other": {
      "by_count": [
       59,
       117,
       70,
       28,
       2
      ],
      "differ": {
       "hitting": {
        "den": 276,
        "n": 140,
        "share": 0.5072463768115942,
        "small": false
       },
       "points": {
        "den": 276,
        "n": 90,
        "share": 0.32608695652173914,
        "small": false
       },
       "prime": {
        "den": 276,
        "n": 77,
        "share": 0.27898550724637683,
        "small": false
       },
       "race": {
        "den": 276,
        "n": 42,
        "share": 0.15217391304347827,
        "small": false
       }
      },
      "gap": {
       "mean": 0.13678256541058637,
       "median": 0.07491430640220642,
       "n": 276
      },
      "loss1": {
       "mean": 0.044575602140115654,
       "median": 0.0,
       "n": 276
      },
      "loss2": {
       "mean": 0.14419909415469653,
       "median": 0.07817256450653076,
       "n": 276
      },
      "n": 276,
      "p1": {
       "mean": 0.6564002342263212,
       "median": 0.67,
       "n": 276
      },
      "p2": {
       "mean": 0.14915166154296589,
       "median": 0.15,
       "n": 276
      },
      "psum": {
       "mean": 0.8055518957692871,
       "median": 0.83,
       "n": 276
      },
      "psum_bins": [
       0,
       0,
       0,
       2,
       3,
       25,
       36,
       56,
       59,
       95
      ]
     }
    },
    "label": "Probabilities over plays, not sampled reasoning.",
    "margins": {
     "main": {
      "classes": {
       "none": {
        "close": {
         "den": 140,
         "n": 61,
         "share": 0.4357142857142857,
         "small": true
        },
        "eligible": 140,
        "risk_close": {
         "den": 61,
         "n": 0,
         "share": 0.0,
         "small": true
        }
       },
       "risk": {
        "close": {
         "den": 175,
         "n": 97,
         "share": 0.5542857142857143,
         "small": true
        },
        "eligible": 175,
        "risk_close": {
         "den": 97,
         "n": 56,
         "share": 0.5773195876288659,
         "small": true
        }
       },
       "safety": {
        "close": {
         "den": 192,
         "n": 73,
         "share": 0.3802083333333333,
         "small": true
        },
        "eligible": 192,
        "risk_close": {
         "den": 73,
         "n": 29,
         "share": 0.3972602739726027,
         "small": true
        }
       }
      },
      "margin": 0.001,
      "missed": {
       "adj_close": 0.5,
       "adj_other": 0.8420289855072464,
       "adjusted": -0.34202898550724636,
       "adjusted_hi": -0.04166666666666674,
       "adjusted_left_out": 0,
       "adjusted_lo": -0.55,
       "close": {
        "den": 56,
        "n": 29,
        "share": 0.5178571428571429,
        "small": true
       },
       "covered": {
        "den": 56,
        "n": 46,
        "share": 0.8214285714285714,
        "small": true
       },
       "crude": -0.31547619047619047,
       "crude_hi": -0.14814814814814814,
       "crude_left_out": 0,
       "crude_lo": -0.4544708777686628,
       "group": 104,
       "matches": 4,
       "na": null,
       "other": {
        "den": 48,
        "n": 40,
        "share": 0.8333333333333334,
        "small": true
       },
       "resamples": 2000,
       "strata": 25,
       "strata_both": 18
      },
      "stake": {
       "big": {
        "den": 95,
        "n": 51,
        "share": 0.5368421052631579,
        "small": true
       },
       "medium": {
        "den": 57,
        "n": 34,
        "share": 0.5964912280701754,
        "small": true
       },
       "small": {
        "den": 23,
        "n": 12,
        "share": 0.5217391304347826,
        "small": true
       }
      }
     },
     "robust": {
      "classes": {
       "none": {
        "close": {
         "den": 140,
         "n": 61,
         "share": 0.4357142857142857,
         "small": true
        },
        "eligible": 140,
        "risk_close": {
         "den": 61,
         "n": 0,
         "share": 0.0,
         "small": true
        }
       },
       "risk": {
        "close": {
         "den": 152,
         "n": 85,
         "share": 0.5592105263157895,
         "small": true
        },
        "eligible": 152,
        "risk_close": {
         "den": 85,
         "n": 48,
         "share": 0.5647058823529412,
         "small": true
        }
       },
       "safety": {
        "close": {
         "den": 215,
         "n": 85,
         "share": 0.3953488372093023,
         "small": false
        },
        "eligible": 215,
        "risk_close": {
         "den": 85,
         "n": 37,
         "share": 0.43529411764705883,
         "small": true
        }
       }
      },
      "margin": 0.02,
      "missed": {
       "adj_close": 0.5263157894736842,
       "adj_other": 0.8114035087719298,
       "adjusted": -0.2850877192982456,
       "adjusted_hi": 0.04999999999999993,
       "adjusted_left_out": 0,
       "adjusted_lo": -0.5333333333333334,
       "close": {
        "den": 48,
        "n": 26,
        "share": 0.5416666666666666,
        "small": true
       },
       "covered": {
        "den": 48,
        "n": 38,
        "share": 0.7916666666666666,
        "small": true
       },
       "crude": -0.28760162601626016,
       "crude_hi": -0.08571428571428574,
       "crude_left_out": 0,
       "crude_lo": -0.4675925925925926,
       "group": 89,
       "matches": 4,
       "na": null,
       "other": {
        "den": 41,
        "n": 34,
        "share": 0.8292682926829268,
        "small": true
       },
       "resamples": 2000,
       "strata": 25,
       "strata_both": 17
      },
      "stake": {
       "big": {
        "den": 95,
        "n": 51,
        "share": 0.5368421052631579,
        "small": true
       },
       "medium": {
        "den": 57,
        "n": 34,
        "share": 0.5964912280701754,
        "small": true
       },
       "small": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       }
      }
     }
    },
    "na": null,
    "risk_close": {
     "den": 231,
     "n": 85,
     "share": 0.36796536796536794,
     "small": false
    },
    "risk_close_of_eligible": {
     "den": 507,
     "n": 85,
     "share": 0.16765285996055226,
     "small": false
    },
    "risk_pair_other": {
     "den": 276,
     "n": 129,
     "share": 0.4673913043478261,
     "small": false
    },
    "ties": {
     "close": 13,
     "member": 28,
     "order": 7,
     "positive": 35,
     "top_two": 35,
     "zero": 0
    }
   },
   "confidence": {
    "auroc_best": 0.7239262214107416,
    "auroc_blunder": 0.671351891278592,
    "confident_badly_wrong": {
     "den": 1910,
     "n": 1,
     "share": 0.0005235602094240838
    },
    "cutoffs": {
     "0.9": {
      "blunders": {
       "den": 51,
       "n": 1,
       "share": 0.0196078431372549
      },
      "kept": {
       "den": 1910,
       "n": 51,
       "share": 0.026701570680628273
      },
      "mean_loss": 0.005219459533691406,
      "too_few": true
     },
     "0.99": {
      "blunders": {
       "den": 1,
       "n": 0,
       "share": 0.0
      },
      "kept": {
       "den": 1910,
       "n": 1,
       "share": 0.0005235602094240838
      },
      "mean_loss": 0.0,
      "too_few": true
     }
    },
    "ece": 0.06461780104712042,
    "held_out": {
     "blunders": 0.018212265362566333,
     "kept": 0.0261045571665124,
     "no_cutoff": 0,
     "runs": 300
    },
    "n": 1910,
    "surest": {
     "cut": 0.68,
     "error_rate": 12.522550133185659,
     "n": 492,
     "too_few": false
    },
    "too_few": false
   },
   "quality": {
    "auroc": 0.7239262214107416,
    "best_share": 0.46234309623430964,
    "blunder_share": 0.24686192468619247,
    "brier": 0.21409842931937173,
    "clusters": 92,
    "confidence_n": 1910,
    "confident_badly_wrong": 0.0005235602094240838,
    "cost_coverage": 1.0,
    "decisions": 1912,
    "ece": 0.06461780104712042,
    "engine_scored": 1912,
    "error_hi": 35.6751832267911,
    "error_lo": 28.254087904925168,
    "error_rate": 31.73452527671429,
    "first_date": "2026-10-19",
    "group_sizes": [
     129,
     164
    ],
    "grouped": 2,
    "invalid": 0,
    "last_date": "2026-10-26",
    "mean_confidence": 0.5274136125654451,
    "over_budget": 0.0,
    "p95_loss": 0.2861639499664306,
    "probability_source": "decision",
    "retry_mean": 0.00035617154811715476,
    "scored": 1912,
    "time_median": 0.192,
    "time_p90": 0.245,
    "tokens_in": 4116.005753138075,
    "tokens_out": 292.1705020920502,
    "too_few": false,
    "usd_per_1000": 0.17287224163179918,
    "very_confident": 0.026701570680628273,
    "worst_mean": 0.4653880450253685
   },
   "risk": {
    "main": {
     "by_call": {
      "missed": 15.996568933690442,
      "needless": 2.8413651768258426,
      "none": 3.891509018495492,
      "right_risk": 1.7213280152950328,
      "right_safety": 4.369847320612505,
      "wrong_risk": 2.913906811794975
     },
     "calls": {
      "missed": 401.0,
      "needless": 78.0,
      "right_risk": 184.0,
      "right_safety": 657.0,
      "wrong_risk": 47.0
     },
     "confidence": {
      "auroc_right_risk": 0.4925869565217391,
      "missed_mean_p": 0.48159600997506236,
      "missed_n": 401,
      "taken_grouped": 0,
      "taken_n": 309,
      "taken_other": 0
     },
     "error_rate": 31.73452527671429,
     "found": {
      "den": 632.0,
      "n": 184.0,
      "share": 0.2911392405063291
     },
     "good": {
      "den": 309.0,
      "n": 184.0,
      "share": 0.5954692556634305
     },
     "judgement": 59.25083957633686,
     "judgement_hi": 61.43527705801678,
     "judgement_lo": 57.12868181655602,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.10491071428571429,
      "x": 0.7088607594936709,
      "y": 0.10612244897959183
     },
     "margin": 0.001,
     "missed": {
      "den": 632.0,
      "n": 401.0,
      "share": 0.634493670886076
     },
     "needless": {
      "den": 735.0,
      "n": 78.0,
      "share": 0.10612244897959183
     },
     "needless_by_loss": {
      "big": {
       "den": 735.0,
       "n": 45.0,
       "share": 0.061224489795918366
      },
      "medium": {
       "den": 735.0,
       "n": 28.0,
       "share": 0.0380952380952381
      },
      "small": {
       "den": 735.0,
       "n": 5.0,
       "share": 0.006802721088435374
      }
     },
     "no_trade_off": 545.0,
     "reference": null,
     "right_safety": {
      "den": 735.0,
      "n": 657.0,
      "share": 0.8938775510204081
     },
     "risk_pays": 632.0,
     "safety_pays": 735.0,
     "scored": 1912,
     "sdt": {
      "aim": {
       "den": 231.0,
       "n": 184.0,
       "share": 0.7965367965367965
      },
      "beta": 1.0619895478526205,
      "caution": 0.7938574357965497,
      "caution_hi": 0.8608233130725628,
      "caution_lo": 0.7292530616512035,
      "cstar": 0.06673131546073889,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 0.9012872054053134,
      "dprime_hi": 1.0720335830013732,
      "dprime_lo": 0.7330396105560286,
      "false_alarm": {
       "den": 735.0,
       "n": 78.0,
       "share": 0.10612244897959183
      },
      "false_alarm_rate": 0.10665760869565218,
      "hit": {
       "den": 632.0,
       "n": 231.0,
       "share": 0.36550632911392406
      },
      "hit_rate": 0.3657187993680885,
      "missed_loss": 0.15254583442003056,
      "missed_n": 401.0,
      "na": null,
      "needless_loss": 0.1392997491818208,
      "needless_n": 78.0
     },
     "size": {
      "bad": {
       "mean": 8.352,
       "n": 125.0,
       "too_few": true
      },
      "right": {
       "mean": 8.706521739130435,
       "n": 184.0,
       "too_few": true
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 355.0,
        "n": 118.0,
        "share": 0.3323943661971831
       },
       "missed": {
        "den": 355.0,
        "n": 210.0,
        "share": 0.5915492957746479
       },
       "risk_pays": 355.0,
       "wrong": {
        "den": 355.0,
        "n": 27.0,
        "share": 0.07605633802816901
       }
      },
      "medium": {
       "found": {
        "den": 204.0,
        "n": 57.0,
        "share": 0.27941176470588236
       },
       "missed": {
        "den": 204.0,
        "n": 133.0,
        "share": 0.6519607843137255
       },
       "risk_pays": 204.0,
       "wrong": {
        "den": 204.0,
        "n": 14.0,
        "share": 0.06862745098039216
       }
      },
      "small": {
       "found": {
        "den": 73.0,
        "n": 9.0,
        "share": 0.1232876712328767
       },
       "missed": {
        "den": 73.0,
        "n": 58.0,
        "share": 0.7945205479452054
       },
       "risk_pays": 73.0,
       "wrong": {
        "den": 73.0,
        "n": 6.0,
        "share": 0.0821917808219178
       }
      }
     },
     "trade_off": 1367.0,
     "wrong": {
      "den": 632.0,
      "n": 47.0,
      "share": 0.07436708860759493
     }
    },
    "robust": {
     "by_call": {
      "missed": 15.639708880101288,
      "needless": 2.9305814215952384,
      "none": 3.891509018495492,
      "right_risk": 1.5250817642675782,
      "right_safety": 4.726707374201659,
      "wrong_risk": 3.020936818053034
     },
     "calls": {
      "missed": 343.0,
      "needless": 93.0,
      "right_risk": 167.0,
      "right_safety": 715.0,
      "wrong_risk": 49.0
     },
     "confidence": {
      "auroc_right_risk": 0.5072109302521717,
      "missed_mean_p": 0.4873760932944606,
      "missed_n": 343,
      "taken_grouped": 0,
      "taken_n": 309,
      "taken_other": 0
     },
     "error_rate": 31.73452527671429,
     "found": {
      "den": 559.0,
      "n": 167.0,
      "share": 0.29874776386404295
     },
     "good": {
      "den": 309.0,
      "n": 167.0,
      "share": 0.540453074433657
     },
     "judgement": 59.18243769815265,
     "judgement_hi": 61.38043666117411,
     "judgement_lo": 56.980673165872005,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.125,
      "x": 0.7012522361359571,
      "y": 0.1150990099009901
     },
     "margin": 0.02,
     "missed": {
      "den": 559.0,
      "n": 343.0,
      "share": 0.6135957066189625
     },
     "needless": {
      "den": 808.0,
      "n": 93.0,
      "share": 0.1150990099009901
     },
     "needless_by_loss": {
      "big": {
       "den": 808.0,
       "n": 46.0,
       "share": 0.05693069306930693
      },
      "medium": {
       "den": 808.0,
       "n": 31.0,
       "share": 0.038366336633663366
      },
      "small": {
       "den": 808.0,
       "n": 16.0,
       "share": 0.019801980198019802
      }
     },
     "no_trade_off": 545.0,
     "reference": null,
     "right_safety": {
      "den": 808.0,
      "n": 715.0,
      "share": 0.8849009900990099
     },
     "risk_pays": 559.0,
     "safety_pays": 808.0,
     "scored": 1912,
     "sdt": {
      "aim": {
       "den": 216.0,
       "n": 167.0,
       "share": 0.7731481481481481
      },
      "beta": 0.9989318298866936,
      "caution": 0.7427879520759149,
      "caution_hi": 0.8108759016907172,
      "caution_lo": 0.6815120993447233,
      "cstar": -0.001175435497548059,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 0.9092298265733927,
      "dprime_hi": 1.0828309200006203,
      "dprime_lo": 0.749303348752545,
      "false_alarm": {
       "den": 808.0,
       "n": 93.0,
       "share": 0.1150990099009901
      },
      "false_alarm_rate": 0.11557478368355995,
      "hit": {
       "den": 559.0,
       "n": 216.0,
       "share": 0.38640429338103754
      },
      "hit_rate": 0.38660714285714287,
      "missed_loss": 0.17436223544462776,
      "missed_n": 343.0,
      "na": null,
      "needless_loss": 0.12050046619548592,
      "needless_n": 93.0
     },
     "size": {
      "bad": {
       "mean": 8.049295774647888,
       "n": 142.0,
       "too_few": true
      },
      "right": {
       "mean": 9.0,
       "n": 167.0,
       "too_few": true
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 355.0,
        "n": 113.0,
        "share": 0.3183098591549296
       },
       "missed": {
        "den": 355.0,
        "n": 210.0,
        "share": 0.5915492957746479
       },
       "risk_pays": 355.0,
       "wrong": {
        "den": 355.0,
        "n": 32.0,
        "share": 0.09014084507042254
       }
      },
      "medium": {
       "found": {
        "den": 204.0,
        "n": 54.0,
        "share": 0.2647058823529412
       },
       "missed": {
        "den": 204.0,
        "n": 133.0,
        "share": 0.6519607843137255
       },
       "risk_pays": 204.0,
       "wrong": {
        "den": 204.0,
        "n": 17.0,
        "share": 0.08333333333333333
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 1367.0,
     "wrong": {
      "den": 559.0,
      "n": 49.0,
      "share": 0.08765652951699464
     }
    }
   },
   "slug": "jev-1-14-0"
  },
  "Liquid d1-ultra": {
   "close": {
    "close": {
     "den": 2070,
     "n": 1262,
     "share": 0.6096618357487923,
     "small": false
    },
    "decisions": 2071,
    "definition": "A decision is a close choice when the second most likely play has at least half the probability of the first (internally: split).",
    "eligible": 2070,
    "exclusions": {
     "grouped": 1
    },
    "groups": {
     "close": {
      "by_count": [
       381,
       469,
       272,
       123,
       17
      ],
      "differ": {
       "hitting": {
        "den": 1262,
        "n": 598,
        "share": 0.47385103011093505,
        "small": false
       },
       "points": {
        "den": 1262,
        "n": 384,
        "share": 0.30427892234548337,
        "small": false
       },
       "prime": {
        "den": 1262,
        "n": 326,
        "share": 0.2583201267828843,
        "small": false
       },
       "race": {
        "den": 1262,
        "n": 142,
        "share": 0.11251980982567353,
        "small": false
       }
      },
      "gap": {
       "mean": 0.10863102601749583,
       "median": 0.05732002854347229,
       "n": 1262
      },
      "loss1": {
       "mean": 0.07743164441192585,
       "median": 0.018425792455673218,
       "n": 1262
      },
      "loss2": {
       "mean": 0.10185215125087702,
       "median": 0.040596649050712585,
       "n": 1262
      },
      "n": 1262,
      "p1": {
       "mean": 0.3012242316636623,
       "median": 0.2729513844887085,
       "n": 1262
      },
      "p2": {
       "mean": 0.21871366468544612,
       "median": 0.19715981914250547,
       "n": 1262
      },
      "psum": {
       "mean": 0.5199378963491085,
       "median": 0.47000632762157873,
       "n": 1262
      },
      "psum_bins": [
       0,
       22,
       132,
       284,
       263,
       171,
       130,
       110,
       58,
       92
      ]
     },
     "other": {
      "by_count": [
       127,
       317,
       253,
       92,
       19
      ],
      "differ": {
       "hitting": {
        "den": 808,
        "n": 451,
        "share": 0.5581683168316832,
        "small": false
       },
       "points": {
        "den": 808,
        "n": 333,
        "share": 0.4121287128712871,
        "small": false
       },
       "prime": {
        "den": 808,
        "n": 265,
        "share": 0.327970297029703,
        "small": false
       },
       "race": {
        "den": 808,
        "n": 126,
        "share": 0.15594059405940594,
        "small": false
       }
      },
      "gap": {
       "mean": 0.14999517997597703,
       "median": 0.07859662175178528,
       "n": 808
      },
      "loss1": {
       "mean": 0.03401301838088744,
       "median": 0.0,
       "n": 808
      },
      "loss2": {
       "mean": 0.15035687394366407,
       "median": 0.07591432332992554,
       "n": 808
      },
      "n": 808,
      "p1": {
       "mean": 0.5473048488400353,
       "median": 0.5458255371502556,
       "n": 808
      },
      "p2": {
       "mean": 0.14743027220109503,
       "median": 0.14001314900973655,
       "n": 808
      },
      "psum": {
       "mean": 0.6947351210411303,
       "median": 0.6929391470855382,
       "n": 808
      },
      "psum_bins": [
       0,
       0,
       10,
       68,
       102,
       107,
       130,
       106,
       115,
       170
      ]
     }
    },
    "label": "Probabilities over plays, not sampled reasoning.",
    "margins": {
     "main": {
      "classes": {
       "none": {
        "close": {
         "den": 567,
         "n": 369,
         "share": 0.6507936507936508,
         "small": false
        },
        "eligible": 567,
        "risk_close": {
         "den": 369,
         "n": 0,
         "share": 0.0,
         "small": false
        }
       },
       "risk": {
        "close": {
         "den": 687,
         "n": 441,
         "share": 0.6419213973799127,
         "small": false
        },
        "eligible": 687,
        "risk_close": {
         "den": 441,
         "n": 267,
         "share": 0.6054421768707483,
         "small": false
        }
       },
       "safety": {
        "close": {
         "den": 816,
         "n": 452,
         "share": 0.553921568627451,
         "small": false
        },
        "eligible": 816,
        "risk_close": {
         "den": 452,
         "n": 223,
         "share": 0.49336283185840707,
         "small": false
        }
       }
      },
      "margin": 0.001,
      "missed": {
       "adj_close": 0.5764705882352941,
       "adj_other": 0.6946358543417367,
       "adjusted": -0.11816526610644262,
       "adjusted_hi": -0.00700909860021539,
       "adjusted_left_out": 0,
       "adjusted_lo": -0.20481328294326306,
       "close": {
        "den": 267,
        "n": 155,
        "share": 0.5805243445692884,
        "small": false
       },
       "covered": {
        "den": 267,
        "n": 255,
        "share": 0.9550561797752809,
        "small": false
       },
       "crude": -0.06392009987515612,
       "crude_hi": 0.032272683824852905,
       "crude_left_out": 0,
       "crude_lo": -0.15447225582998234,
       "group": 402,
       "matches": 18,
       "na": null,
       "other": {
        "den": 135,
        "n": 87,
        "share": 0.6444444444444445,
        "small": true
       },
       "resamples": 2000,
       "strata": 35,
       "strata_both": 26
      },
      "stake": {
       "big": {
        "den": 371,
        "n": 254,
        "share": 0.6846361185983828,
        "small": false
       },
       "medium": {
        "den": 235,
        "n": 140,
        "share": 0.5957446808510638,
        "small": false
       },
       "small": {
        "den": 81,
        "n": 47,
        "share": 0.5802469135802469,
        "small": true
       }
      }
     },
     "robust": {
      "classes": {
       "none": {
        "close": {
         "den": 567,
         "n": 369,
         "share": 0.6507936507936508,
         "small": false
        },
        "eligible": 567,
        "risk_close": {
         "den": 369,
         "n": 0,
         "share": 0.0,
         "small": false
        }
       },
       "risk": {
        "close": {
         "den": 606,
         "n": 394,
         "share": 0.6501650165016502,
         "small": false
        },
        "eligible": 606,
        "risk_close": {
         "den": 394,
         "n": 243,
         "share": 0.616751269035533,
         "small": false
        }
       },
       "safety": {
        "close": {
         "den": 897,
         "n": 499,
         "share": 0.5562987736900781,
         "small": false
        },
        "eligible": 897,
        "risk_close": {
         "den": 499,
         "n": 247,
         "share": 0.49498997995991983,
         "small": false
        }
       }
      },
      "margin": 0.02,
      "missed": {
       "adj_close": 0.5673076923076923,
       "adj_other": 0.6431757478632479,
       "adjusted": -0.07586805555555565,
       "adjusted_hi": 0.041883902872555695,
       "adjusted_left_out": 0,
       "adjusted_lo": -0.18926672363308303,
       "close": {
        "den": 243,
        "n": 140,
        "share": 0.5761316872427984,
        "small": false
       },
       "covered": {
        "den": 243,
        "n": 208,
        "share": 0.8559670781893004,
        "small": false
       },
       "crude": -0.06275720164609044,
       "crude_hi": 0.04835247338055347,
       "crude_left_out": 0,
       "crude_lo": -0.17114699287018156,
       "group": 351,
       "matches": 18,
       "na": null,
       "other": {
        "den": 108,
        "n": 69,
        "share": 0.6388888888888888,
        "small": true
       },
       "resamples": 2000,
       "strata": 34,
       "strata_both": 24
      },
      "stake": {
       "big": {
        "den": 371,
        "n": 254,
        "share": 0.6846361185983828,
        "small": false
       },
       "medium": {
        "den": 235,
        "n": 140,
        "share": 0.5957446808510638,
        "small": false
       },
       "small": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       }
      }
     }
    },
    "na": null,
    "risk_close": {
     "den": 1262,
     "n": 490,
     "share": 0.38827258320126784,
     "small": false
    },
    "risk_close_of_eligible": {
     "den": 2070,
     "n": 490,
     "share": 0.23671497584541062,
     "small": false
    },
    "risk_pair_other": {
     "den": 808,
     "n": 380,
     "share": 0.47029702970297027,
     "small": false
    },
    "ties": {
     "close": 116,
     "member": 82,
     "order": 60,
     "positive": 142,
     "top_two": 142,
     "zero": 0
    }
   },
   "confidence": {
    "auroc_best": 0.7463826066159435,
    "auroc_blunder": 0.6979319688027945,
    "confident_badly_wrong": {
     "den": 2070,
     "n": 0,
     "share": 0.0
    },
    "cutoffs": {
     "0.9": {
      "blunders": {
       "den": 20,
       "n": 0,
       "share": 0.0
      },
      "kept": {
       "den": 2070,
       "n": 20,
       "share": 0.00966183574879227
      },
      "mean_loss": 0.0006012499332427978,
      "too_few": true
     },
     "0.99": {
      "blunders": {
       "den": 0,
       "n": 0,
       "share": null
      },
      "kept": {
       "den": 2070,
       "n": 0,
       "share": 0.0
      },
      "mean_loss": null,
      "too_few": true
     }
    },
    "ece": 0.08871145013415928,
    "held_out": {
     "blunders": 0.027983979504483503,
     "kept": 0.022074620131283616,
     "no_cutoff": 0,
     "runs": 300
    },
    "n": 2070,
    "surest": {
     "cut": 0.5246623165651004,
     "error_rate": 8.605671903261772,
     "n": 520,
     "too_few": false
    },
    "too_few": false
   },
   "quality": {
    "auroc": 0.7463826066159435,
    "best_share": 0.4857556735876388,
    "blunder_share": 0.22404635441815549,
    "brier": 0.21372114742457327,
    "clusters": 102,
    "confidence_n": 2070,
    "confident_badly_wrong": 0.0,
    "cost_coverage": 1.0,
    "decisions": 2071,
    "ece": 0.08871145013415928,
    "engine_scored": 2071,
    "error_hi": 33.37192323780477,
    "error_lo": 27.290356978630378,
    "error_rate": 30.33737738255658,
    "first_date": "2026-10-19",
    "group_sizes": [
     142
    ],
    "grouped": 1,
    "invalid": 0,
    "last_date": "2026-10-25",
    "mean_confidence": 0.3972788880300919,
    "over_budget": 0.0,
    "p95_loss": 0.28908631205558777,
    "probability_source": "decision",
    "retry_mean": 0.0005678416224046355,
    "scored": 2071,
    "time_median": 0.233,
    "time_p90": 0.412,
    "tokens_in": 3461.4939642684694,
    "tokens_out": 0.0,
    "too_few": false,
    "usd_per_1000": 0.1453827464992757,
    "very_confident": 0.00966183574879227,
    "worst_mean": 0.48681695128862673
   },
   "risk": {
    "main": {
     "by_call": {
      "missed": 13.253726263025543,
      "needless": 3.66484415180035,
      "none": 3.5112518898131824,
      "right_risk": 2.3714831950640343,
      "right_safety": 4.516391852580079,
      "wrong_risk": 3.0196800302733897
     },
     "calls": {
      "missed": 341.0,
      "needless": 114.0,
      "right_risk": 289.0,
      "right_safety": 702.0,
      "wrong_risk": 58.0
     },
     "confidence": {
      "auroc_right_risk": 0.6363965558863764,
      "missed_mean_p": 0.3417520511955815,
      "missed_n": 340,
      "taken_grouped": 0,
      "taken_n": 461,
      "taken_other": 0
     },
     "error_rate": 30.33737738255658,
     "found": {
      "den": 688.0,
      "n": 289.0,
      "share": 0.42005813953488375
     },
     "good": {
      "den": 461.0,
      "n": 289.0,
      "share": 0.6268980477223427
     },
     "judgement": 64.01761285909713,
     "judgement_hi": 66.32900926829026,
     "judgement_lo": 61.74463607411275,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.14536340852130325,
      "x": 0.5799418604651163,
      "y": 0.13970588235294118
     },
     "margin": 0.001,
     "missed": {
      "den": 688.0,
      "n": 341.0,
      "share": 0.4956395348837209
     },
     "needless": {
      "den": 816.0,
      "n": 114.0,
      "share": 0.13970588235294118
     },
     "needless_by_loss": {
      "big": {
       "den": 816.0,
       "n": 60.0,
       "share": 0.07352941176470588
      },
      "medium": {
       "den": 816.0,
       "n": 39.0,
       "share": 0.04779411764705882
      },
      "small": {
       "den": 816.0,
       "n": 15.0,
       "share": 0.01838235294117647
      }
     },
     "no_trade_off": 567.0,
     "reference": null,
     "right_safety": {
      "den": 816.0,
      "n": 702.0,
      "share": 0.8602941176470589
     },
     "risk_pays": 688.0,
     "safety_pays": 816.0,
     "scored": 2071,
     "sdt": {
      "aim": {
       "den": 347.0,
       "n": 289.0,
       "share": 0.8328530259365994
      },
      "beta": 0.9809994468087577,
      "caution": 0.5343726278307428,
      "caution_hi": 0.6113653595688409,
      "caution_lo": 0.4634320009589807,
      "cstar": -0.017590169675579692,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.0905740920174787,
      "dprime_hi": 1.2625135257875657,
      "dprime_lo": 0.9215588976820086,
      "false_alarm": {
       "den": 816.0,
       "n": 114.0,
       "share": 0.13970588235294118
      },
      "false_alarm_rate": 0.1401468788249694,
      "hit": {
       "den": 688.0,
       "n": 347.0,
       "share": 0.5043604651162791
      },
      "hit_rate": 0.5043541364296081,
      "missed_loss": 0.160988076778451,
      "missed_n": 341.0,
      "na": null,
      "needless_loss": 0.13315600418207937,
      "needless_n": 114.0
     },
     "size": {
      "bad": {
       "mean": 10.581395348837209,
       "n": 172.0,
       "too_few": true
      },
      "right": {
       "mean": 9.892733564013842,
       "n": 289.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 371.0,
        "n": 180.0,
        "share": 0.48517520215633425
       },
       "missed": {
        "den": 371.0,
        "n": 168.0,
        "share": 0.4528301886792453
       },
       "risk_pays": 371.0,
       "wrong": {
        "den": 371.0,
        "n": 23.0,
        "share": 0.06199460916442048
       }
      },
      "medium": {
       "found": {
        "den": 236.0,
        "n": 88.0,
        "share": 0.3728813559322034
       },
       "missed": {
        "den": 236.0,
        "n": 120.0,
        "share": 0.5084745762711864
       },
       "risk_pays": 236.0,
       "wrong": {
        "den": 236.0,
        "n": 28.0,
        "share": 0.11864406779661017
       }
      },
      "small": {
       "found": {
        "den": 81.0,
        "n": 21.0,
        "share": 0.25925925925925924
       },
       "missed": {
        "den": 81.0,
        "n": 53.0,
        "share": 0.654320987654321
       },
       "risk_pays": 81.0,
       "wrong": {
        "den": 81.0,
        "n": 7.0,
        "share": 0.08641975308641975
       }
      }
     },
     "trade_off": 1504.0,
     "wrong": {
      "den": 688.0,
      "n": 58.0,
      "share": 0.08430232558139535
     }
    },
    "robust": {
     "by_call": {
      "missed": 12.83954510592079,
      "needless": 4.187164935209163,
      "none": 3.5112518898131824,
      "right_risk": 1.8430128153944176,
      "right_safety": 4.930573009684833,
      "wrong_risk": 3.0258296265341933
     },
     "calls": {
      "missed": 288.0,
      "needless": 142.0,
      "right_risk": 254.0,
      "right_safety": 755.0,
      "wrong_risk": 65.0
     },
     "confidence": {
      "auroc_right_risk": 0.6175016166457453,
      "missed_mean_p": 0.3389476602975704,
      "missed_n": 287,
      "taken_grouped": 0,
      "taken_n": 461,
      "taken_other": 0
     },
     "error_rate": 30.33737738255658,
     "found": {
      "den": 607.0,
      "n": 254.0,
      "share": 0.4184514003294893
     },
     "good": {
      "den": 461.0,
      "n": 254.0,
      "share": 0.5509761388286334
     },
     "judgement": 63.00729688381003,
     "judgement_hi": 65.34656697528615,
     "judgement_lo": 60.77377423253403,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.18413597733711048,
      "x": 0.5815485996705108,
      "y": 0.15830546265328874
     },
     "margin": 0.02,
     "missed": {
      "den": 607.0,
      "n": 288.0,
      "share": 0.4744645799011532
     },
     "needless": {
      "den": 897.0,
      "n": 142.0,
      "share": 0.15830546265328874
     },
     "needless_by_loss": {
      "big": {
       "den": 897.0,
       "n": 63.0,
       "share": 0.07023411371237458
      },
      "medium": {
       "den": 897.0,
       "n": 42.0,
       "share": 0.046822742474916385
      },
      "small": {
       "den": 897.0,
       "n": 37.0,
       "share": 0.04124860646599777
      }
     },
     "no_trade_off": 567.0,
     "reference": null,
     "right_safety": {
      "den": 897.0,
      "n": 755.0,
      "share": 0.8416945373467113
     },
     "risk_pays": 607.0,
     "safety_pays": 897.0,
     "scored": 2071,
     "sdt": {
      "aim": {
       "den": 319.0,
       "n": 254.0,
       "share": 0.7962382445141066
      },
      "beta": 0.9774134847050919,
      "caution": 0.46796349401260934,
      "caution_hi": 0.5404508510622944,
      "caution_lo": 0.3980829785554694,
      "cstar": -0.021474982491911838,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.063819155323055,
      "dprime_hi": 1.2236158132076218,
      "dprime_lo": 0.9079433398374129,
      "false_alarm": {
       "den": 897.0,
       "n": 142.0,
       "share": 0.15830546265328874
      },
      "false_alarm_rate": 0.15868596881959912,
      "hit": {
       "den": 607.0,
       "n": 319.0,
       "share": 0.5255354200988468
      },
      "hit_rate": 0.5254934210526315,
      "missed_loss": 0.18465762440529135,
      "missed_n": 288.0,
      "na": null,
      "needless_loss": 0.12213547296927009,
      "needless_n": 142.0
     },
     "size": {
      "bad": {
       "mean": 10.719806763285025,
       "n": 207.0,
       "too_few": false
      },
      "right": {
       "mean": 9.68503937007874,
       "n": 254.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 371.0,
        "n": 172.0,
        "share": 0.4636118598382749
       },
       "missed": {
        "den": 371.0,
        "n": 168.0,
        "share": 0.4528301886792453
       },
       "risk_pays": 371.0,
       "wrong": {
        "den": 371.0,
        "n": 31.0,
        "share": 0.08355795148247978
       }
      },
      "medium": {
       "found": {
        "den": 236.0,
        "n": 82.0,
        "share": 0.3474576271186441
       },
       "missed": {
        "den": 236.0,
        "n": 120.0,
        "share": 0.5084745762711864
       },
       "risk_pays": 236.0,
       "wrong": {
        "den": 236.0,
        "n": 34.0,
        "share": 0.1440677966101695
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 1504.0,
     "wrong": {
      "den": 607.0,
      "n": 65.0,
      "share": 0.1070840197693575
     }
    }
   },
   "slug": "liquid-d1-ultra"
  },
  "Qwen3.8 Flash (off)": {
   "close": {
    "close": {
     "den": 615,
     "n": 248,
     "share": 0.4032520325203252,
     "small": false
    },
    "decisions": 2027,
    "definition": "A decision is a close choice when the second most likely play has at least half the probability of the first (internally: split).",
    "eligible": 615,
    "exclusions": {
     "grouped": 145,
     "not_audited": 243,
     "sparse": 1024
    },
    "groups": {
     "close": {
      "by_count": [
       136,
       76,
       29,
       7,
       0
      ],
      "differ": {
       "hitting": {
        "den": 248,
        "n": 72,
        "share": 0.2903225806451613,
        "small": false
       },
       "points": {
        "den": 248,
        "n": 26,
        "share": 0.10483870967741936,
        "small": false
       },
       "prime": {
        "den": 248,
        "n": 42,
        "share": 0.1693548387096774,
        "small": false
       },
       "race": {
        "den": 248,
        "n": 15,
        "share": 0.06048387096774194,
        "small": false
       }
      },
      "gap": {
       "mean": 0.07306473305629145,
       "median": 0.020918577909469604,
       "n": 248
      },
      "loss1": {
       "mean": 0.04429457192459414,
       "median": 0.0005459189414978027,
       "n": 248
      },
      "loss2": {
       "mean": 0.057688062109293475,
       "median": 0.0025146007537841797,
       "n": 248
      },
      "n": 248,
      "p1": {
       "mean": 0.4964752320322249,
       "median": 0.5,
       "n": 248
      },
      "p2": {
       "mean": 0.3535581711617629,
       "median": 0.3486451285660856,
       "n": 248
      },
      "psum": {
       "mean": 0.8500334031939877,
       "median": 0.8545939524298262,
       "n": 248
      },
      "psum_bins": [
       0,
       0,
       0,
       0,
       1,
       4,
       26,
       59,
       60,
       98
      ]
     },
     "other": {
      "by_count": [
       121,
       163,
       76,
       6,
       1
      ],
      "differ": {
       "hitting": {
        "den": 367,
        "n": 144,
        "share": 0.3923705722070845,
        "small": false
       },
       "points": {
        "den": 367,
        "n": 48,
        "share": 0.1307901907356948,
        "small": false
       },
       "prime": {
        "den": 367,
        "n": 102,
        "share": 0.2779291553133515,
        "small": false
       },
       "race": {
        "den": 367,
        "n": 43,
        "share": 0.11716621253405994,
        "small": false
       }
      },
      "gap": {
       "mean": 0.12355472443863871,
       "median": 0.0481795072555542,
       "n": 367
      },
      "loss1": {
       "mean": 0.02577661558457876,
       "median": 0.0,
       "n": 367
      },
      "loss2": {
       "mean": 0.10937601279172976,
       "median": 0.029506266117095947,
       "n": 367
      },
      "n": 367,
      "p1": {
       "mean": 0.7198372182785608,
       "median": 0.7310585786300049,
       "n": 367
      },
      "p2": {
       "mean": 0.1915670094152328,
       "median": 0.19409981065351464,
       "n": 367
      },
      "psum": {
       "mean": 0.9114042276937936,
       "median": 0.9190675420371415,
       "n": 367
      },
      "psum_bins": [
       0,
       0,
       0,
       0,
       0,
       0,
       4,
       37,
       110,
       216
      ]
     }
    },
    "label": "Probabilities over plays, not sampled reasoning.",
    "margins": {
     "main": {
      "classes": {
       "none": {
        "close": {
         "den": 343,
         "n": 152,
         "share": 0.44314868804664725,
         "small": false
        },
        "eligible": 343,
        "risk_close": {
         "den": 152,
         "n": 0,
         "share": 0.0,
         "small": true
        }
       },
       "risk": {
        "close": {
         "den": 83,
         "n": 36,
         "share": 0.43373493975903615,
         "small": true
        },
        "eligible": 83,
        "risk_close": {
         "den": 36,
         "n": 27,
         "share": 0.75,
         "small": true
        }
       },
       "safety": {
        "close": {
         "den": 189,
         "n": 60,
         "share": 0.31746031746031744,
         "small": true
        },
        "eligible": 189,
        "risk_close": {
         "den": 60,
         "n": 45,
         "share": 0.75,
         "small": true
        }
       }
      },
      "margin": 0.001,
      "missed": {
       "adj_close": 0.7407407407407407,
       "adj_other": 0.8812636165577342,
       "adjusted": -0.14052287581699352,
       "adjusted_hi": 0.09389903329752955,
       "adjusted_left_out": 0,
       "adjusted_lo": -0.3793103448275862,
       "close": {
        "den": 27,
        "n": 20,
        "share": 0.7407407407407407,
        "small": true
       },
       "covered": {
        "den": 27,
        "n": 27,
        "share": 1.0,
        "small": true
       },
       "crude": -0.15399610136452246,
       "crude_hi": 0.05556418219461695,
       "crude_left_out": 0,
       "crude_lo": -0.3393692528735632,
       "group": 65,
       "matches": 16,
       "na": null,
       "other": {
        "den": 38,
        "n": 34,
        "share": 0.8947368421052632,
        "small": true
       },
       "resamples": 2000,
       "strata": 7,
       "strata_both": 4
      },
      "stake": {
       "big": {
        "den": 35,
        "n": 16,
        "share": 0.45714285714285713,
        "small": true
       },
       "medium": {
        "den": 35,
        "n": 13,
        "share": 0.37142857142857144,
        "small": true
       },
       "small": {
        "den": 13,
        "n": 7,
        "share": 0.5384615384615384,
        "small": true
       }
      }
     },
     "robust": {
      "classes": {
       "none": {
        "close": {
         "den": 343,
         "n": 152,
         "share": 0.44314868804664725,
         "small": false
        },
        "eligible": 343,
        "risk_close": {
         "den": 152,
         "n": 0,
         "share": 0.0,
         "small": true
        }
       },
       "risk": {
        "close": {
         "den": 70,
         "n": 29,
         "share": 0.4142857142857143,
         "small": true
        },
        "eligible": 70,
        "risk_close": {
         "den": 29,
         "n": 23,
         "share": 0.7931034482758621,
         "small": true
        }
       },
       "safety": {
        "close": {
         "den": 202,
         "n": 67,
         "share": 0.3316831683168317,
         "small": false
        },
        "eligible": 202,
        "risk_close": {
         "den": 67,
         "n": 49,
         "share": 0.7313432835820896,
         "small": true
        }
       }
      },
      "margin": 0.02,
      "missed": {
       "adj_close": 0.6956521739130435,
       "adj_other": 0.860613810741688,
       "adjusted": -0.1649616368286445,
       "adjusted_hi": 0.1071546509046509,
       "adjusted_left_out": 0,
       "adjusted_lo": -0.4285714285714286,
       "close": {
        "den": 23,
        "n": 16,
        "share": 0.6956521739130435,
        "small": true
       },
       "covered": {
        "den": 23,
        "n": 23,
        "share": 1.0,
        "small": true
       },
       "crude": -0.17934782608695654,
       "crude_hi": 0.048593911248709966,
       "crude_left_out": 0,
       "crude_lo": -0.38095238095238093,
       "group": 55,
       "matches": 16,
       "na": null,
       "other": {
        "den": 32,
        "n": 28,
        "share": 0.875,
        "small": true
       },
       "resamples": 2000,
       "strata": 6,
       "strata_both": 3
      },
      "stake": {
       "big": {
        "den": 35,
        "n": 16,
        "share": 0.45714285714285713,
        "small": true
       },
       "medium": {
        "den": 35,
        "n": 13,
        "share": 0.37142857142857144,
        "small": true
       },
       "small": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       }
      }
     }
    },
    "na": null,
    "risk_close": {
     "den": 248,
     "n": 72,
     "share": 0.2903225806451613,
     "small": false
    },
    "risk_close_of_eligible": {
     "den": 615,
     "n": 72,
     "share": 0.11707317073170732,
     "small": false
    },
    "risk_pair_other": {
     "den": 367,
     "n": 143,
     "share": 0.3896457765667575,
     "small": false
    },
    "ties": {
     "close": 30,
     "member": 21,
     "order": 25,
     "positive": 46,
     "top_two": 46,
     "zero": 0
    }
   },
   "confidence": {
    "auroc_best": 0.6902031303510336,
    "auroc_blunder": 0.64391240595178,
    "confident_badly_wrong": {
     "den": 1856,
     "n": 1,
     "share": 0.0005387931034482759
    },
    "cutoffs": {
     "0.9": {
      "blunders": {
       "den": 17,
       "n": 1,
       "share": 0.058823529411764705
      },
      "kept": {
       "den": 1856,
       "n": 17,
       "share": 0.009159482758620689
      },
      "mean_loss": 0.01728734198738547,
      "too_few": true
     },
     "0.99": {
      "blunders": {
       "den": 0,
       "n": 0,
       "share": null
      },
      "kept": {
       "den": 1856,
       "n": 0,
       "share": 0.0
      },
      "mean_loss": null,
      "too_few": true
     }
    },
    "ece": 0.10663676591127538,
    "held_out": {
     "blunders": 0.04170700750315758,
     "kept": 0.014867679062048346,
     "no_cutoff": 0,
     "runs": 300
    },
    "n": 1856,
    "surest": {
     "cut": 0.6936623301396077,
     "error_rate": 15.111620537936687,
     "n": 464,
     "too_few": false
    },
    "too_few": false
   },
   "quality": {
    "auroc": 0.6902031303510336,
    "best_share": 0.44647261963492846,
    "blunder_share": 0.2525900345337938,
    "brier": 0.23245478786226023,
    "clusters": 100,
    "confidence_n": 1856,
    "confident_badly_wrong": 0.0005387931034482759,
    "cost_coverage": 1.0,
    "decisions": 2027,
    "ece": 0.10663676591127538,
    "engine_scored": 2027,
    "error_hi": 36.72076523712284,
    "error_lo": 30.827352191678397,
    "error_rate": 33.577662334397445,
    "first_date": "2026-10-20",
    "group_sizes": [
     14,
     15,
     16,
     17,
     18,
     19,
     20,
     21,
     22,
     23,
     24,
     25,
     26
    ],
    "grouped": 171,
    "invalid": 0,
    "last_date": "2026-10-26",
    "mean_confidence": 0.5662272831526548,
    "over_budget": 0.0029600394671928957,
    "p95_loss": 0.3082065761089321,
    "probability_source": "logprobs",
    "retry_mean": 8.970692649235323,
    "scored": 2027,
    "time_median": 1.699,
    "time_p90": 7.4746000000000015,
    "tokens_in": 4181.5259003453375,
    "tokens_out": 10.232856438085841,
    "too_few": false,
    "usd_per_1000": 0.28444589738529846,
    "very_confident": 0.009159482758620689,
    "worst_mean": 0.5329444119245699
   },
   "risk": {
    "main": {
     "by_call": {
      "missed": 14.390172801643507,
      "needless": 4.643808761781581,
      "none": 4.167065678644956,
      "right_risk": 3.459256684538997,
      "right_safety": 2.9609489379166734,
      "wrong_risk": 3.956409469871728
     },
     "calls": {
      "missed": 372.0,
      "needless": 112.0,
      "right_risk": 243.0,
      "right_safety": 613.0,
      "wrong_risk": 91.0
     },
     "confidence": {
      "auroc_right_risk": 0.5723981900452488,
      "missed_mean_p": 0.5452550439711948,
      "missed_n": 328,
      "taken_grouped": 60,
      "taken_n": 386,
      "taken_other": 0
     },
     "error_rate": 33.577662334397445,
     "found": {
      "den": 706.0,
      "n": 243.0,
      "share": 0.34419263456090654
     },
     "good": {
      "den": 446.0,
      "n": 243.0,
      "share": 0.5448430493273543
     },
     "judgement": 59.48549379701085,
     "judgement_hi": 61.664850018420324,
     "judgement_lo": 57.18966824246997,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.19654427645788336,
      "x": 0.6558073654390935,
      "y": 0.15448275862068966
     },
     "margin": 0.001,
     "missed": {
      "den": 706.0,
      "n": 372.0,
      "share": 0.5269121813031161
     },
     "needless": {
      "den": 725.0,
      "n": 112.0,
      "share": 0.15448275862068966
     },
     "needless_by_loss": {
      "big": {
       "den": 725.0,
       "n": 69.0,
       "share": 0.09517241379310344
      },
      "medium": {
       "den": 725.0,
       "n": 27.0,
       "share": 0.037241379310344824
      },
      "small": {
       "den": 725.0,
       "n": 16.0,
       "share": 0.022068965517241378
      }
     },
     "no_trade_off": 596.0,
     "reference": null,
     "right_safety": {
      "den": 725.0,
      "n": 613.0,
      "share": 0.8455172413793104
     },
     "risk_pays": 706.0,
     "safety_pays": 725.0,
     "scored": 2027,
     "sdt": {
      "aim": {
       "den": 334.0,
       "n": 243.0,
       "share": 0.7275449101796407
      },
      "beta": 1.1006938508090325,
      "caution": 0.5414049546791285,
      "caution_hi": 0.6244238992685485,
      "caution_lo": 0.4607700346844515,
      "cstar": 0.10120535425190852,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 0.9479810146636443,
      "dprime_hi": 1.1217167436682967,
      "dprime_lo": 0.787142596421905,
      "false_alarm": {
       "den": 725.0,
       "n": 112.0,
       "share": 0.15448275862068966
      },
      "false_alarm_rate": 0.15495867768595042,
      "hit": {
       "den": 706.0,
       "n": 334.0,
       "share": 0.4730878186968839
      },
      "hit_rate": 0.47312588401697314,
      "missed_loss": 0.15682193692973864,
      "missed_n": 372.0,
      "na": null,
      "needless_loss": 0.16808929214520113,
      "needless_n": 112.0
     },
     "size": {
      "bad": {
       "mean": 9.482758620689655,
       "n": 203.0,
       "too_few": false
      },
      "right": {
       "mean": 7.716049382716049,
       "n": 243.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 391.0,
        "n": 166.0,
        "share": 0.42455242966751916
       },
       "missed": {
        "den": 391.0,
        "n": 190.0,
        "share": 0.4859335038363171
       },
       "risk_pays": 391.0,
       "wrong": {
        "den": 391.0,
        "n": 35.0,
        "share": 0.08951406649616368
       }
      },
      "medium": {
       "found": {
        "den": 234.0,
        "n": 66.0,
        "share": 0.28205128205128205
       },
       "missed": {
        "den": 234.0,
        "n": 126.0,
        "share": 0.5384615384615384
       },
       "risk_pays": 234.0,
       "wrong": {
        "den": 234.0,
        "n": 42.0,
        "share": 0.1794871794871795
       }
      },
      "small": {
       "found": {
        "den": 81.0,
        "n": 11.0,
        "share": 0.13580246913580246
       },
       "missed": {
        "den": 81.0,
        "n": 56.0,
        "share": 0.691358024691358
       },
       "risk_pays": 81.0,
       "wrong": {
        "den": 81.0,
        "n": 14.0,
        "share": 0.1728395061728395
       }
      }
     },
     "trade_off": 1431.0,
     "wrong": {
      "den": 706.0,
      "n": 91.0,
      "share": 0.12889518413597734
     }
    },
    "robust": {
     "by_call": {
      "missed": 14.052466028537665,
      "needless": 4.9764906310305905,
      "none": 4.167065678644956,
      "right_risk": 3.0523541971092807,
      "right_safety": 3.2986557110225148,
      "wrong_risk": 4.030630088052434
     },
     "calls": {
      "missed": 316.0,
      "needless": 137.0,
      "right_risk": 222.0,
      "right_safety": 669.0,
      "wrong_risk": 87.0
     },
     "confidence": {
      "auroc_right_risk": 0.5686856743611104,
      "missed_mean_p": 0.5490316532020334,
      "missed_n": 281,
      "taken_grouped": 60,
      "taken_n": 386,
      "taken_other": 0
     },
     "error_rate": 33.577662334397445,
     "found": {
      "den": 625.0,
      "n": 222.0,
      "share": 0.3552
     },
     "good": {
      "den": 446.0,
      "n": 222.0,
      "share": 0.4977578475336323
     },
     "judgement": 59.26124069478909,
     "judgement_hi": 61.516045691280944,
     "judgement_lo": 57.08963467847478,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.21588089330024815,
      "x": 0.6448,
      "y": 0.16997518610421836
     },
     "margin": 0.02,
     "missed": {
      "den": 625.0,
      "n": 316.0,
      "share": 0.5056
     },
     "needless": {
      "den": 806.0,
      "n": 137.0,
      "share": 0.16997518610421836
     },
     "needless_by_loss": {
      "big": {
       "den": 806.0,
       "n": 78.0,
       "share": 0.0967741935483871
      },
      "medium": {
       "den": 806.0,
       "n": 31.0,
       "share": 0.038461538461538464
      },
      "small": {
       "den": 806.0,
       "n": 28.0,
       "share": 0.034739454094292806
      }
     },
     "no_trade_off": 596.0,
     "reference": null,
     "right_safety": {
      "den": 806.0,
      "n": 669.0,
      "share": 0.8300248138957816
     },
     "risk_pays": 625.0,
     "safety_pays": 806.0,
     "scored": 2027,
     "sdt": {
      "aim": {
       "den": 309.0,
       "n": 222.0,
       "share": 0.7184466019417476
      },
      "beta": 1.0533972654805093,
      "caution": 0.4833317453111749,
      "caution_hi": 0.5597613948697244,
      "caution_lo": 0.40891548059283717,
      "cstar": 0.0554214714412974,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 0.9386331833569682,
      "dprime_hi": 1.0841427518063445,
      "dprime_lo": 0.7938767826292001,
      "false_alarm": {
       "den": 806.0,
       "n": 137.0,
       "share": 0.16997518610421836
      },
      "false_alarm_rate": 0.17038413878562578,
      "hit": {
       "den": 625.0,
       "n": 309.0,
       "share": 0.4944
      },
      "hit_rate": 0.49440894568690097,
      "missed_loss": 0.18028068759396107,
      "missed_n": 316.0,
      "na": null,
      "needless_loss": 0.14726053297954755,
      "needless_n": 137.0
     },
     "size": {
      "bad": {
       "mean": 9.178571428571429,
       "n": 224.0,
       "too_few": false
      },
      "right": {
       "mean": 7.8558558558558556,
       "n": 222.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 391.0,
        "n": 160.0,
        "share": 0.4092071611253197
       },
       "missed": {
        "den": 391.0,
        "n": 190.0,
        "share": 0.4859335038363171
       },
       "risk_pays": 391.0,
       "wrong": {
        "den": 391.0,
        "n": 41.0,
        "share": 0.10485933503836317
       }
      },
      "medium": {
       "found": {
        "den": 234.0,
        "n": 62.0,
        "share": 0.26495726495726496
       },
       "missed": {
        "den": 234.0,
        "n": 126.0,
        "share": 0.5384615384615384
       },
       "risk_pays": 234.0,
       "wrong": {
        "den": 234.0,
        "n": 46.0,
        "share": 0.19658119658119658
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 1431.0,
     "wrong": {
      "den": 625.0,
      "n": 87.0,
      "share": 0.1392
     }
    }
   },
   "slug": "qwen3-8-flash-off"
  },
  "Qwen3.8-27B (low)": {
   "close": {
    "close": {
     "den": 298,
     "n": 0,
     "share": 0.0,
     "small": false
    },
    "decisions": 2108,
    "definition": "A decision is a close choice when the second most likely play has at least half the probability of the first (internally: split).",
    "eligible": 298,
    "exclusions": {
     "grouped": 142,
     "not_audited": 729,
     "sparse": 939
    },
    "groups": {
     "close": {
      "by_count": [
       0,
       0,
       0,
       0,
       0
      ],
      "differ": {
       "hitting": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       },
       "points": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       },
       "prime": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       },
       "race": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       }
      },
      "gap": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "loss1": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "loss2": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "n": 0,
      "p1": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "p2": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "psum": {
       "mean": null,
       "median": null,
       "n": 0
      },
      "psum_bins": [
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0
      ]
     },
     "other": {
      "by_count": [
       96,
       138,
       50,
       14,
       0
      ],
      "differ": {
       "hitting": {
        "den": 298,
        "n": 105,
        "share": 0.3523489932885906,
        "small": false
       },
       "points": {
        "den": 298,
        "n": 46,
        "share": 0.15436241610738255,
        "small": false
       },
       "prime": {
        "den": 298,
        "n": 90,
        "share": 0.30201342281879195,
        "small": false
       },
       "race": {
        "den": 298,
        "n": 39,
        "share": 0.13087248322147652,
        "small": false
       }
      },
      "gap": {
       "mean": 0.11986740023497768,
       "median": 0.05388367176055908,
       "n": 298
      },
      "loss1": {
       "mean": 0.03075728120419803,
       "median": 0.0,
       "n": 298
      },
      "loss2": {
       "mean": 0.11208530060396898,
       "median": 0.039676934480667114,
       "n": 298
      },
      "n": 298,
      "p1": {
       "mean": 0.9998077571462516,
       "median": 0.9998911007789086,
       "n": 298
      },
      "p2": {
       "mean": 0.0001625422826910124,
       "median": 9.170061060868894e-05,
       "n": 298
      },
      "psum": {
       "mean": 0.9999702994289426,
       "median": 0.9999896722444813,
       "n": 298
      },
      "psum_bins": [
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       0,
       298
      ]
     }
    },
    "label": "Probabilities over plays, not sampled reasoning.",
    "margins": {
     "main": {
      "classes": {
       "none": {
        "close": {
         "den": 175,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 175,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       },
       "risk": {
        "close": {
         "den": 33,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 33,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       },
       "safety": {
        "close": {
         "den": 90,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 90,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       }
      },
      "margin": 0.001,
      "missed": {
       "close": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       },
       "group": 24,
       "na": "N/A: no close choices in this group",
       "other": {
        "den": 24,
        "n": 17,
        "share": 0.7083333333333334,
        "small": true
       }
      },
      "stake": {
       "big": {
        "den": 16,
        "n": 0,
        "share": 0.0,
        "small": true
       },
       "medium": {
        "den": 14,
        "n": 0,
        "share": 0.0,
        "small": true
       },
       "small": {
        "den": 3,
        "n": 0,
        "share": 0.0,
        "small": true
       }
      }
     },
     "robust": {
      "classes": {
       "none": {
        "close": {
         "den": 175,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 175,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       },
       "risk": {
        "close": {
         "den": 30,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 30,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       },
       "safety": {
        "close": {
         "den": 93,
         "n": 0,
         "share": 0.0,
         "small": true
        },
        "eligible": 93,
        "risk_close": {
         "den": 0,
         "n": 0,
         "share": null,
         "small": false
        }
       }
      },
      "margin": 0.02,
      "missed": {
       "close": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       },
       "group": 23,
       "na": "N/A: no close choices in this group",
       "other": {
        "den": 23,
        "n": 16,
        "share": 0.6956521739130435,
        "small": true
       }
      },
      "stake": {
       "big": {
        "den": 16,
        "n": 0,
        "share": 0.0,
        "small": true
       },
       "medium": {
        "den": 14,
        "n": 0,
        "share": 0.0,
        "small": true
       },
       "small": {
        "den": 0,
        "n": 0,
        "share": null,
        "small": false
       }
      }
     }
    },
    "na": null,
    "risk_close": {
     "den": 0,
     "n": 0,
     "share": null,
     "small": false
    },
    "risk_close_of_eligible": {
     "den": 298,
     "n": 0,
     "share": 0.0,
     "small": false
    },
    "risk_pair_other": {
     "den": 298,
     "n": 102,
     "share": 0.3422818791946309,
     "small": false
    },
    "ties": {
     "close": 0,
     "member": 0,
     "order": 0,
     "positive": 0,
     "top_two": 0,
     "zero": 0
    }
   },
   "confidence": {
    "auroc_best": 0.5712672678108771,
    "auroc_blunder": 0.5513074402786994,
    "confident_badly_wrong": {
     "den": 1891,
     "n": 384,
     "share": 0.20306716023268112
    },
    "cutoffs": {
     "0.9": {
      "blunders": {
       "den": 1891,
       "n": 384,
       "share": 0.20306716023268112
      },
      "kept": {
       "den": 1891,
       "n": 1891,
       "share": 1.0
      },
      "mean_loss": 0.05327173701863261,
      "too_few": false
     },
     "0.99": {
      "blunders": {
       "den": 1889,
       "n": 383,
       "share": 0.2027527792482795
      },
      "kept": {
       "den": 1891,
       "n": 1889,
       "share": 0.9989423585404548
      },
      "mean_loss": 0.05326914334499123,
      "too_few": false
     }
    },
    "ece": 0.44654850944977653,
    "held_out": {
     "blunders": 0.7416506557996329,
     "kept": 0.004721407118234083,
     "no_cutoff": 200,
     "runs": 300
    },
    "n": 1891,
    "surest": {
     "cut": 0.9999038976031153,
     "error_rate": 22.285679493091575,
     "n": 473,
     "too_few": false
    },
    "too_few": false
   },
   "quality": {
    "auroc": 0.5712672678108771,
    "best_share": 0.5275142314990512,
    "blunder_share": 0.2239089184060721,
    "brier": 0.4465850974578028,
    "clusters": 98,
    "confidence_n": 1891,
    "confident_badly_wrong": 0.20306716023268112,
    "cost_coverage": 1.0,
    "decisions": 2108,
    "ece": 0.44654850944977653,
    "engine_scored": 2108,
    "error_hi": 31.81670804432192,
    "error_lo": 26.542902621482654,
    "error_rate": 29.032554555210716,
    "first_date": "2026-10-19",
    "group_sizes": [
     14,
     15,
     16,
     17,
     18,
     19,
     20,
     21,
     22,
     23,
     24,
     25,
     26
    ],
    "grouped": 217,
    "invalid": 0,
    "last_date": "2026-10-26",
    "mean_confidence": 0.9996949927919235,
    "over_budget": 0.9767552182163188,
    "own_cost": {
     "decisions": 2108,
     "estimate": true,
     "excluded_hours": 122.01666666666667,
     "gaps": [
      [
       "2026-10-20T07:20Z",
       "2026-10-20T20:17Z",
       777
      ],
      [
       "2026-10-20T21:06Z",
       "2026-10-20T21:34Z",
       28
      ],
      [
       "2026-10-20T21:52Z",
       "2026-10-20T22:17Z",
       25
      ],
      [
       "2026-10-20T22:19Z",
       "2026-10-20T22:33Z",
       14
      ],
      [
       "2026-10-20T22:33Z",
       "2026-10-20T22:47Z",
       14
      ],
      [
       "2026-10-20T22:51Z",
       "2026-10-20T23:16Z",
       25
      ],
      [
       "2026-10-20T23:46Z",
       "2026-10-20T23:56Z",
       10
      ],
      [
       "2026-10-20T23:56Z",
       "2026-10-21T01:40Z",
       104
      ],
      [
       "2026-10-21T02:12Z",
       "2026-10-21T02:22Z",
       10
      ],
      [
       "2026-10-21T02:22Z",
       "2026-10-21T02:37Z",
       15
      ],
      [
       "2026-10-21T04:13Z",
       "2026-10-21T04:34Z",
       21
      ],
      [
       "2026-10-21T04:55Z",
       "2026-10-21T06:51Z",
       115
      ],
      [
       "2026-10-21T06:51Z",
       "2026-10-21T07:15Z",
       25
      ],
      [
       "2026-10-21T07:15Z",
       "2026-10-21T07:46Z",
       30
      ],
      [
       "2026-10-21T07:55Z",
       "2026-10-21T08:23Z",
       28
      ],
      [
       "2026-10-21T08:42Z",
       "2026-10-21T09:05Z",
       23
      ],
      [
       "2026-10-21T09:17Z",
       "2026-10-21T09:30Z",
       12
      ],
      [
       "2026-10-21T09:30Z",
       "2026-10-21T09:46Z",
       16
      ],
      [
       "2026-10-21T09:46Z",
       "2026-10-21T10:29Z",
       43
      ],
      [
       "2026-10-21T10:29Z",
       "2026-10-21T10:51Z",
       22
      ],
      [
       "2026-10-21T10:51Z",
       "2026-10-21T11:08Z",
       17
      ],
      [
       "2026-10-21T11:08Z",
       "2026-10-21T11:46Z",
       38
      ],
      [
       "2026-10-21T11:49Z",
       "2026-10-21T12:06Z",
       16
      ],
      [
       "2026-10-21T12:06Z",
       "2026-10-21T12:20Z",
       15
      ],
      [
       "2026-10-21T12:20Z",
       "2026-10-21T12:34Z",
       14
      ],
      [
       "2026-10-21T12:45Z",
       "2026-10-21T13:01Z",
       16
      ],
      [
       "2026-10-21T13:01Z",
       "2026-10-21T13:14Z",
       13
      ],
      [
       "2026-10-21T13:14Z",
       "2026-10-21T13:26Z",
       12
      ],
      [
       "2026-10-21T13:26Z",
       "2026-10-21T22:19Z",
       533
      ],
      [
       "2026-10-21T22:27Z",
       "2026-10-21T22:49Z",
       22
      ],
      [
       "2026-10-21T22:49Z",
       "2026-10-21T23:42Z",
       54
      ],
      [
       "2026-10-21T23:49Z",
       "2026-10-22T00:08Z",
       19
      ],
      [
       "2026-10-22T00:35Z",
       "2026-10-22T00:52Z",
       18
      ],
      [
       "2026-10-22T00:52Z",
       "2026-10-22T01:04Z",
       12
      ],
      [
       "2026-10-22T01:40Z",
       "2026-10-22T01:52Z",
       12
      ],
      [
       "2026-10-22T02:00Z",
       "2026-10-22T02:11Z",
       11
      ],
      [
       "2026-10-22T02:51Z",
       "2026-10-22T03:05Z",
       14
      ],
      [
       "2026-10-22T03:12Z",
       "2026-10-22T03:24Z",
       12
      ],
      [
       "2026-10-22T03:28Z",
       "2026-10-22T03:40Z",
       12
      ],
      [
       "2026-10-22T03:43Z",
       "2026-10-22T03:58Z",
       15
      ],
      [
       "2026-10-22T04:11Z",
       "2026-10-22T04:32Z",
       21
      ],
      [
       "2026-10-22T04:32Z",
       "2026-10-22T04:49Z",
       17
      ],
      [
       "2026-10-22T05:20Z",
       "2026-10-22T05:31Z",
       11
      ],
      [
       "2026-10-22T05:31Z",
       "2026-10-22T05:48Z",
       18
      ],
      [
       "2026-10-22T05:48Z",
       "2026-10-22T06:07Z",
       19
      ],
      [
       "2026-10-22T06:12Z",
       "2026-10-22T15:21Z",
       548
      ],
      [
       "2026-10-22T15:24Z",
       "2026-10-23T21:42Z",
       1818
      ],
      [
       "2026-10-23T23:09Z",
       "2026-10-24T06:51Z",
       462
      ],
      [
       "2026-10-24T12:27Z",
       "2026-10-24T12:37Z",
       11
      ],
      [
       "2026-10-24T12:42Z",
       "2026-10-25T09:59Z",
       1277
      ],
      [
       "2026-10-25T11:45Z",
       "2026-10-25T12:00Z",
       15
      ],
      [
       "2026-10-25T12:17Z",
       "2026-10-25T12:48Z",
       31
      ],
      [
       "2026-10-25T13:06Z",
       "2026-10-25T13:57Z",
       51
      ],
      [
       "2026-10-25T18:48Z",
       "2026-10-25T18:59Z",
       11
      ],
      [
       "2026-10-25T19:00Z",
       "2026-10-25T20:18Z",
       78
      ],
      [
       "2026-10-25T20:43Z",
       "2026-10-25T21:56Z",
       73
      ],
      [
       "2026-10-25T23:48Z",
       "2026-10-26T00:10Z",
       22
      ],
      [
       "2026-10-26T01:21Z",
       "2026-10-26T10:36Z",
       556
      ],
      [
       "2026-10-26T10:40Z",
       "2026-10-26T10:54Z",
       14
      ],
      [
       "2026-10-26T12:19Z",
       "2026-10-26T12:43Z",
       24
      ],
      [
       "2026-10-26T12:46Z",
       "2026-10-26T12:59Z",
       12
      ]
     ],
     "gpu_decisions": 2107,
     "gpu_usd": 82.25680152382861,
     "hosted_decisions": 1,
     "hosted_unpriced": 0,
     "hosted_usd": 0.004513,
     "hours": 37.62484944429661,
     "kind": "gpu",
     "parts": [
      {
       "gpus": 1,
       "hours": 5.01895444439517,
       "machine": "dgx_spark",
       "usd": 1.068122974074034,
       "usd_per_gpu_hour": 0.21281782608464228
      },
      {
       "gpus": 1,
       "hours": 32.60589499990144,
       "machine": "rtx_pro_6000",
       "usd": 81.18867854975458,
       "usd_per_gpu_hour": 2.49
      }
     ],
     "provenance": "content/own_cost.json: wall clock on our machines × hourly rate",
     "spark_rate": {
      "chf_per_h": 0.17465558152207,
      "electricity_chf_per_h": 0.02244858,
      "purchase_chf_per_h": 0.15220700152207,
      "usd_per_h": 0.21281782608464228
     },
     "usd": 82.26131452382862,
     "usd_per_1000": 39.023393986635966
    },
    "p95_loss": 0.2902082860469816,
    "probability_source": "logprobs",
    "retry_mean": 23.978749525616696,
    "scored": 2108,
    "time_median": 171.0915,
    "time_p90": 566.6976,
    "tokens_in": 14722.56356736243,
    "tokens_out": 5771.840607210626,
    "too_few": false,
    "usd_per_1000": 39.023393986635966,
    "very_confident": 1.0,
    "worst_mean": 0.4630539329279037
   },
   "risk": {
    "main": {
     "by_call": {
      "missed": 14.818232329792044,
      "needless": 2.894927579662153,
      "none": 3.427488980302322,
      "right_risk": 1.9358618633796865,
      "right_safety": 3.406015106351371,
      "wrong_risk": 2.5500286957231375
     },
     "calls": {
      "missed": 417.0,
      "needless": 80.0,
      "right_risk": 242.0,
      "right_safety": 735.0,
      "wrong_risk": 56.0
     },
     "confidence": {
      "auroc_right_risk": 0.5551182667461737,
      "missed_mean_p": 0.9996869026632875,
      "missed_n": 355,
      "taken_grouped": 46,
      "taken_n": 332,
      "taken_other": 0
     },
     "error_rate": 29.032554555210716,
     "found": {
      "den": 715.0,
      "n": 242.0,
      "share": 0.3384615384615385
     },
     "good": {
      "den": 378.0,
      "n": 242.0,
      "share": 0.6402116402116402
     },
     "judgement": 62.01510146295423,
     "judgement_hi": 63.92380641362988,
     "judgement_lo": 60.05998713903362,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.11839323467230443,
      "x": 0.6615384615384615,
      "y": 0.09815950920245399
     },
     "margin": 0.001,
     "missed": {
      "den": 715.0,
      "n": 417.0,
      "share": 0.5832167832167832
     },
     "needless": {
      "den": 815.0,
      "n": 80.0,
      "share": 0.09815950920245399
     },
     "needless_by_loss": {
      "big": {
       "den": 815.0,
       "n": 49.0,
       "share": 0.06012269938650307
      },
      "medium": {
       "den": 815.0,
       "n": 18.0,
       "share": 0.022085889570552148
      },
      "small": {
       "den": 815.0,
       "n": 13.0,
       "share": 0.015950920245398775
      }
     },
     "no_trade_off": 578.0,
     "reference": null,
     "right_safety": {
      "den": 815.0,
      "n": 735.0,
      "share": 0.901840490797546
     },
     "risk_pays": 715.0,
     "safety_pays": 815.0,
     "scored": 2108,
     "sdt": {
      "aim": {
       "den": 298.0,
       "n": 242.0,
       "share": 0.8120805369127517
      },
      "beta": 1.16075065935685,
      "caution": 0.7495513983053473,
      "caution_hi": 0.8425033955148096,
      "caution_lo": 0.6644386605950552,
      "cstar": 0.13809665353431372,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 1.079439017140998,
      "dprime_hi": 1.2396977012270294,
      "dprime_lo": 0.9167106924692962,
      "false_alarm": {
       "den": 815.0,
       "n": 80.0,
       "share": 0.09815950920245399
      },
      "false_alarm_rate": 0.09865196078431372,
      "hit": {
       "den": 715.0,
       "n": 298.0,
       "share": 0.4167832167832168
      },
      "hit_rate": 0.4168994413407821,
      "missed_loss": 0.1498169484470102,
      "missed_n": 417.0,
      "na": null,
      "needless_loss": 0.15256268344819546,
      "needless_n": 80.0
     },
     "size": {
      "bad": {
       "mean": 8.955882352941176,
       "n": 136.0,
       "too_few": true
      },
      "right": {
       "mean": 10.045454545454545,
       "n": 242.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 412.0,
        "n": 174.0,
        "share": 0.4223300970873786
       },
       "missed": {
        "den": 412.0,
        "n": 219.0,
        "share": 0.5315533980582524
       },
       "risk_pays": 412.0,
       "wrong": {
        "den": 412.0,
        "n": 19.0,
        "share": 0.04611650485436893
       }
      },
      "medium": {
       "found": {
        "den": 218.0,
        "n": 54.0,
        "share": 0.24770642201834864
       },
       "missed": {
        "den": 218.0,
        "n": 142.0,
        "share": 0.6513761467889908
       },
       "risk_pays": 218.0,
       "wrong": {
        "den": 218.0,
        "n": 22.0,
        "share": 0.10091743119266056
       }
      },
      "small": {
       "found": {
        "den": 85.0,
        "n": 14.0,
        "share": 0.16470588235294117
       },
       "missed": {
        "den": 85.0,
        "n": 56.0,
        "share": 0.6588235294117647
       },
       "risk_pays": 85.0,
       "wrong": {
        "den": 85.0,
        "n": 15.0,
        "share": 0.17647058823529413
       }
      }
     },
     "trade_off": 1530.0,
     "wrong": {
      "den": 715.0,
      "n": 56.0,
      "share": 0.07832167832167833
     }
    },
    "robust": {
     "by_call": {
      "missed": 14.460080248468062,
      "needless": 3.394332850454428,
      "none": 3.427488980302322,
      "right_risk": 1.5604385727949341,
      "right_safety": 3.764167187675353,
      "wrong_risk": 2.4260467155156147
     },
     "calls": {
      "missed": 361.0,
      "needless": 109.0,
      "right_risk": 217.0,
      "right_safety": 791.0,
      "wrong_risk": 52.0
     },
     "confidence": {
      "auroc_right_risk": 0.5294778214486243,
      "missed_mean_p": 0.9996923304545018,
      "missed_n": 311,
      "taken_grouped": 46,
      "taken_n": 332,
      "taken_other": 0
     },
     "error_rate": 29.032554555210716,
     "found": {
      "den": 630.0,
      "n": 217.0,
      "share": 0.34444444444444444
     },
     "good": {
      "den": 378.0,
      "n": 217.0,
      "share": 0.5740740740740741
     },
     "judgement": 61.16666666666667,
     "judgement_hi": 63.22447190050626,
     "judgement_lo": 59.15618291433863,
     "judgement_na": null,
     "map": {
      "wrong_share": 0.12590799031476999,
      "x": 0.6555555555555556,
      "y": 0.12111111111111111
     },
     "margin": 0.02,
     "missed": {
      "den": 630.0,
      "n": 361.0,
      "share": 0.573015873015873
     },
     "needless": {
      "den": 900.0,
      "n": 109.0,
      "share": 0.12111111111111111
     },
     "needless_by_loss": {
      "big": {
       "den": 900.0,
       "n": 55.0,
       "share": 0.06111111111111111
      },
      "medium": {
       "den": 900.0,
       "n": 26.0,
       "share": 0.028888888888888888
      },
      "small": {
       "den": 900.0,
       "n": 28.0,
       "share": 0.03111111111111111
      }
     },
     "no_trade_off": 578.0,
     "reference": null,
     "right_safety": {
      "den": 900.0,
      "n": 791.0,
      "share": 0.8788888888888889
     },
     "risk_pays": 630.0,
     "safety_pays": 900.0,
     "scored": 2108,
     "sdt": {
      "aim": {
       "den": 269.0,
       "n": 217.0,
       "share": 0.8066914498141264
      },
      "beta": 1.110622296714257,
      "caution": 0.6755634823307199,
      "caution_hi": 0.7668794460817858,
      "caution_lo": 0.5887643920767562,
      "cstar": 0.10666968436799164,
      "cstar_note": "unstable: fewer than 200 needless risks behind a mean cost",
      "cstar_status": "unstable",
      "dprime": 0.9836017283017907,
      "dprime_hi": 1.148553327513359,
      "dprime_lo": 0.8264402929426503,
      "false_alarm": {
       "den": 900.0,
       "n": 109.0,
       "share": 0.12111111111111111
      },
      "false_alarm_rate": 0.12153163152053274,
      "hit": {
       "den": 630.0,
       "n": 269.0,
       "share": 0.42698412698412697
      },
      "hit_rate": 0.42709984152139463,
      "missed_loss": 0.1688745106026076,
      "missed_n": 361.0,
      "na": null,
      "needless_loss": 0.13128905777537495,
      "needless_n": 109.0
     },
     "size": {
      "bad": {
       "mean": 8.875776397515528,
       "n": 161.0,
       "too_few": true
      },
      "right": {
       "mean": 10.23041474654378,
       "n": 217.0,
       "too_few": false
      }
     },
     "stake": {
      "big": {
       "found": {
        "den": 412.0,
        "n": 166.0,
        "share": 0.4029126213592233
       },
       "missed": {
        "den": 412.0,
        "n": 219.0,
        "share": 0.5315533980582524
       },
       "risk_pays": 412.0,
       "wrong": {
        "den": 412.0,
        "n": 27.0,
        "share": 0.06553398058252427
       }
      },
      "medium": {
       "found": {
        "den": 218.0,
        "n": 51.0,
        "share": 0.23394495412844038
       },
       "missed": {
        "den": 218.0,
        "n": 142.0,
        "share": 0.6513761467889908
       },
       "risk_pays": 218.0,
       "wrong": {
        "den": 218.0,
        "n": 25.0,
        "share": 0.11467889908256881
       }
      },
      "small": {
       "found": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "missed": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       },
       "risk_pays": 0.0,
       "wrong": {
        "den": 0.0,
        "n": 0.0,
        "share": null
       }
      }
     },
     "trade_off": 1530.0,
     "wrong": {
      "den": 630.0,
      "n": 52.0,
      "share": 0.08253968253968254
     }
    }
   },
   "slug": "qwen3-8-27b-on"
  }
 },
 "standings": [
  {
   "drawn": 2,
   "games_lost": 39,
   "games_won": 63,
   "gammons_won": 22,
   "lost": 2,
   "name": "Gemini 3.8 Flash (low)",
   "place": 1,
   "played": 18,
   "points": 44,
   "to_play": 0,
   "won": 14
  },
  {
   "drawn": 1,
   "games_lost": 41,
   "games_won": 65,
   "gammons_won": 17,
   "lost": 4,
   "name": "Claude Sonnet 5.5 (low)",
   "place": 2,
   "played": 18,
   "points": 40,
   "to_play": 0,
   "won": 13
  },
  {
   "drawn": 0,
   "games_lost": 42,
   "games_won": 62,
   "gammons_won": 11,
   "lost": 5,
   "name": "Gemma 4 31B (on)",
   "place": 3,
   "played": 18,
   "points": 39,
   "to_play": 0,
   "won": 13
  },
  {
   "drawn": 1,
   "games_lost": 46,
   "games_won": 62,
   "gammons_won": 23,
   "lost": 5,
   "name": "DeepSeek-V4.1-Flash 1017 (low)",
   "place": 4,
   "played": 18,
   "points": 37,
   "to_play": 0,
   "won": 12
  },
  {
   "drawn": 0,
   "games_lost": 41,
   "games_won": 61,
   "gammons_won": 11,
   "lost": 6,
   "name": "Claude Sonnet 5.5 (high)",
   "place": 5,
   "played": 18,
   "points": 36,
   "to_play": 0,
   "won": 12
  },
  {
   "drawn": 0,
   "games_lost": 43,
   "games_won": 59,
   "gammons_won": 15,
   "lost": 6,
   "name": "DeepSeek-V4.1-Flash (low)",
   "place": 6,
   "played": 18,
   "points": 36,
   "to_play": 0,
   "won": 12
  },
  {
   "drawn": 0,
   "games_lost": 47,
   "games_won": 57,
   "gammons_won": 11,
   "lost": 6,
   "name": "Gemini 3.8 Flash (high)",
   "place": 7,
   "played": 18,
   "points": 36,
   "to_play": 0,
   "won": 12
  },
  {
   "drawn": 0,
   "games_lost": 43,
   "games_won": 59,
   "gammons_won": 14,
   "lost": 7,
   "name": "Grok 4.7 (low)",
   "place": 8,
   "played": 18,
   "points": 33,
   "to_play": 0,
   "won": 11
  },
  {
   "drawn": 0,
   "games_lost": 42,
   "games_won": 56,
   "gammons_won": 13,
   "lost": 8,
   "name": "Claude Fable 5.1 (low)",
   "place": 9,
   "played": 18,
   "points": 30,
   "to_play": 0,
   "won": 10
  },
  {
   "drawn": 0,
   "games_lost": 42,
   "games_won": 52,
   "gammons_won": 14,
   "lost": 8,
   "name": "Gemma 4 31B (off)",
   "place": 10,
   "played": 18,
   "points": 30,
   "to_play": 0,
   "won": 10
  },
  {
   "drawn": 0,
   "games_lost": 45,
   "games_won": 53,
   "gammons_won": 6,
   "lost": 9,
   "name": "Qwen3.8-27B (low)",
   "place": 11,
   "played": 18,
   "points": 27,
   "to_play": 0,
   "won": 9
  },
  {
   "drawn": 0,
   "games_lost": 52,
   "games_won": 42,
   "gammons_won": 21,
   "lost": 10,
   "name": "GNUbg weak",
   "place": 12,
   "played": 18,
   "points": 24,
   "to_play": 0,
   "won": 8
  },
  {
   "drawn": 0,
   "games_lost": 61,
   "games_won": 39,
   "gammons_won": 12,
   "lost": 11,
   "name": "GLM-5.3 (low)",
   "place": 13,
   "played": 18,
   "points": 21,
   "to_play": 0,
   "won": 7
  },
  {
   "drawn": 1,
   "games_lost": 56,
   "games_won": 46,
   "gammons_won": 10,
   "lost": 11,
   "name": "Liquid d1-ultra",
   "place": 14,
   "played": 18,
   "points": 19,
   "to_play": 0,
   "won": 6
  },
  {
   "drawn": 1,
   "games_lost": 63,
   "games_won": 39,
   "gammons_won": 13,
   "lost": 12,
   "name": "DeepSeek-V4.1-Flash 1017 (off)",
   "place": 15,
   "played": 18,
   "points": 16,
   "to_play": 0,
   "won": 5
  },
  {
   "drawn": 0,
   "games_lost": 59,
   "games_won": 45,
   "gammons_won": 9,
   "lost": 13,
   "name": "Fastino GLiDE",
   "place": 16,
   "played": 18,
   "points": 15,
   "to_play": 0,
   "won": 5
  },
  {
   "drawn": 0,
   "games_lost": 58,
   "games_won": 34,
   "gammons_won": 8,
   "lost": 13,
   "name": "Jev 1.14.0",
   "place": 17,
   "played": 18,
   "points": 15,
   "to_play": 0,
   "won": 5
  },
  {
   "drawn": 0,
   "games_lost": 67,
   "games_won": 33,
   "gammons_won": 8,
   "lost": 16,
   "name": "Qwen3.8 Flash (off)",
   "place": 18,
   "played": 18,
   "points": 6,
   "to_play": 0,
   "won": 2
  },
  {
   "drawn": 0,
   "games_lost": 67,
   "games_won": 27,
   "gammons_won": 8,
   "lost": 16,
   "name": "Clef",
   "place": 19,
   "played": 18,
   "points": 6,
   "to_play": 0,
   "won": 2
  }
 ],
 "started": "2026-10-19T12:25:45.795Z",
 "state": "finished",
 "status": "Finished",
 "version": "v3",
 "winner": "Gemini 3.8 Flash (low)"
}
