{
 "dataset": "smoke-part-a",
 "rows": [
  {
   "answered": 50,
   "asked": 50,
   "best_play": 0.68,
   "blunders": 0.12,
   "error_ci": [
    5.36361962556839,
    24.348824322223663
   ],
   "error_positions": 50,
   "error_rate": 13.571449220180511,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "0a97729e+dirty"
   ],
   "invalid": 0,
   "paired": [
    {
     "ci": [
      0.0,
      0.0
     ],
     "diff": 0.0,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      -2.866925001144409,
      15.085244178771973
     ],
     "diff": 5.039750933647156,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "none (choice only)",
   "provider": "antigravity-subscription",
   "quantization": null,
   "reasoning": "low",
   "request_s_p50": 2.8745,
   "request_s_p90": 6.0236,
   "retry_s_max": 0.0,
   "retry_s_mean": 0.0,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "Gemini 3.8 Flash (low) [antigravity]",
   "spec": "antigravity/gemini-3.8-flash",
   "timed_out": 0,
   "tokens_in": 3244.46,
   "tokens_out": 275.8,
   "tokens_reasoning": 272.8,
   "usd_per_1000": 3.4675949999999998,
   "usd_total": 0.17337975
  },
  {
   "answered": 50,
   "asked": 50,
   "best_play": 0.54,
   "blunders": 0.18,
   "error_ci": [
    9.099238514900208,
    27.961522936820984
   ],
   "error_positions": 50,
   "error_rate": 17.999955415725708,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "0a97729e+dirty"
   ],
   "invalid": 0,
   "paired": [
    {
     "ci": [
      -7.125324606895447,
      15.192023813724518
     ],
     "diff": 4.4285061955451965,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      2.670912444591522,
      17.745797634124756
     ],
     "diff": 9.468257129192352,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "none (choice only)",
   "provider": "claude-subscription",
   "quantization": null,
   "reasoning": "low",
   "request_s_p50": 1.354,
   "request_s_p90": 1.6473000000000002,
   "retry_s_max": 0.0,
   "retry_s_mean": 0.0,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "Claude Haiku 5.5 (low)",
   "spec": "anthropic/claude-haiku-5-5",
   "timed_out": 0,
   "tokens_in": 4270.82,
   "tokens_out": 20.18,
   "tokens_reasoning": 9.16,
   "usd_per_1000": 0.437172,
   "usd_total": 0.021858600000000002
  },
  {
   "answered": 50,
   "asked": 50,
   "best_play": 0.6,
   "blunders": 0.18,
   "error_ci": [
    10.127369165420532,
    33.87252151966095
   ],
   "error_positions": 50,
   "error_rate": 21.050951182842255,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "0a97729e+dirty"
   ],
   "invalid": 0,
   "paired": [
    {
     "ci": [
      -0.6032228469848633,
      17.111780643463135
     ],
     "diff": 7.479501962661743,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      3.0146753787994385,
      23.955979347229004
     ],
     "diff": 12.519252896308899,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "none (choice only)",
   "provider": "claude-subscription",
   "quantization": null,
   "reasoning": "high",
   "request_s_p50": 3.1955,
   "request_s_p90": 6.2161,
   "retry_s_max": 0.0,
   "retry_s_mean": 0.0,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "Claude Haiku 5.5 (high)",
   "spec": "anthropic/claude-haiku-5-5",
   "timed_out": 0,
   "tokens_in": 4270.82,
   "tokens_out": 526.44,
   "tokens_reasoning": 515.42,
   "usd_per_1000": 0.6903020000000001,
   "usd_total": 0.0345151
  },
  {
   "answered": 50,
   "asked": 50,
   "best_play": 0.68,
   "blunders": 0.1,
   "error_ci": [
    4.783344268798828,
    20.485351085662842
   ],
   "error_positions": 50,
   "error_rate": 11.712368726730347,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "0a97729e+dirty"
   ],
   "invalid": 0,
   "paired": [
    {
     "ci": [
      -12.664914727210999,
      7.774648666381836
     ],
     "diff": -1.8590804934501648,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      -0.6989514827728271,
      8.646160364151001
     ],
     "diff": 3.180670440196991,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "none (choice only)",
   "provider": "claude-subscription",
   "quantization": null,
   "reasoning": "low",
   "request_s_p50": 1.678,
   "request_s_p90": 2.5120000000000005,
   "retry_s_max": 0.0,
   "retry_s_mean": 0.0,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "Claude Fable 5.1 (low)",
   "spec": "anthropic/claude-fable-5-1",
   "timed_out": 0,
   "tokens_in": 4270.82,
   "tokens_out": 82.46,
   "tokens_reasoning": 71.44,
   "usd_per_1000": 46.831199999999995,
   "usd_total": 2.34156
  },
  {
   "answered": 50,
   "asked": 50,
   "best_play": 0.74,
   "blunders": 0.06,
   "error_ci": [
    2.180524170398712,
    18.47550004720688
   ],
   "error_positions": 50,
   "error_rate": 9.036160111427307,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "0a97729e+dirty"
   ],
   "invalid": 0,
   "paired": [
    {
     "ci": [
      -9.206952452659607,
      -0.9578612446784973
     ],
     "diff": -4.535289108753204,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      -6.624714136123657,
      9.57005500793457
     ],
     "diff": 0.5044618248939514,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "none (choice only)",
   "provider": "claude-subscription",
   "quantization": null,
   "reasoning": "high",
   "request_s_p50": 6.6579999999999995,
   "request_s_p90": 30.790100000000002,
   "retry_s_max": 0.0,
   "retry_s_mean": 0.0,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "Claude Fable 5.1 (high)",
   "spec": "anthropic/claude-fable-5-1",
   "timed_out": 0,
   "tokens_in": 4270.82,
   "tokens_out": 719.08,
   "tokens_reasoning": 708.06,
   "usd_per_1000": 78.6622,
   "usd_total": 3.93311
  },
  {
   "answered": 50,
   "asked": 50,
   "best_play": 0.4489795918367347,
   "blunders": 0.24489795918367346,
   "error_ci": [
    12.762706498710477,
    34.3061059105153
   ],
   "error_positions": 49,
   "error_rate": 22.882575587350495,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "0a97729e+dirty"
   ],
   "invalid": 1,
   "paired": [
    {
     "ci": [
      -3.5065120580245037,
      21.35446211513208
     ],
     "diff": 9.034158015737729,
     "positions": 49,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      6.4694002574803875,
      23.937863050674906
     ],
     "diff": 14.176761009255234,
     "positions": 49,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "none (choice only)",
   "provider": "openrouter:inception",
   "quantization": null,
   "reasoning": "off",
   "request_s_p50": 0.5395000000000001,
   "request_s_p90": 1.497200000000001,
   "retry_s_max": 0.0,
   "retry_s_mean": 0.0,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "Mercury 3 (off)",
   "spec": "openai-compatible/inception/mercury-3",
   "timed_out": 0,
   "tokens_in": 2990.06,
   "tokens_out": 4.04,
   "tokens_reasoning": 0,
   "usd_per_1000": 0.2885742,
   "usd_total": 0.014428709999999999
  },
  {
   "answered": 50,
   "asked": 50,
   "best_play": 0.6,
   "blunders": 0.2222222222222222,
   "error_ci": [
    10.479452874925402,
    38.972290025817024
   ],
   "error_positions": 45,
   "error_rate": 23.615743385420906,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "0a97729e+dirty"
   ],
   "invalid": 5,
   "paired": [
    {
     "ci": [
      1.9715401861402724,
      26.01056893666585
     ],
     "diff": 12.173951996697319,
     "positions": 45,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      5.451251400841607,
      33.74652134047614
     ],
     "diff": 17.77367525630527,
     "positions": 45,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "none (choice only)",
   "provider": "openrouter:inception",
   "quantization": null,
   "reasoning": "low",
   "request_s_p50": 23.003500000000003,
   "request_s_p90": 78.7431,
   "retry_s_max": 0.0,
   "retry_s_mean": 0.0,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "Mercury 3 (low)",
   "spec": "openai-compatible/inception/mercury-3",
   "timed_out": 0,
   "tokens_in": 2986.74,
   "tokens_out": 13524.38,
   "tokens_reasoning": 13518.86,
   "usd_per_1000": 4.8475432000000005,
   "usd_total": 0.24237716
  },
  {
   "answered": 50,
   "asked": 50,
   "best_play": 0.56,
   "blunders": 0.22,
   "error_ci": [
    11.859777569770813,
    42.92833983898163
   ],
   "error_positions": 50,
   "error_rate": 25.8986097574234,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "0a97729e+dirty"
   ],
   "invalid": 0,
   "paired": [
    {
     "ci": [
      -2.6072847843170166,
      29.061796069145203
     ],
     "diff": 12.32716053724289,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      5.803651809692383,
      32.22181499004364
     ],
     "diff": 17.366911470890045,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "none (choice only)",
   "provider": "openrouter:xiaomi/fp8",
   "quantization": "fp8",
   "reasoning": "off",
   "request_s_p50": 2.2055,
   "request_s_p90": 3.3615000000000004,
   "retry_s_max": 0.0,
   "retry_s_mean": 0.0,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "MiMo V2.6 Pro (off)",
   "spec": "openai-compatible/xiaomi/mimo-v2.6-pro",
   "timed_out": 0,
   "tokens_in": 3209.02,
   "tokens_out": 8.26,
   "tokens_reasoning": 0,
   "usd_per_1000": 0.638876172,
   "usd_total": 0.0319438086
  },
  {
   "answered": 42,
   "asked": 50,
   "best_play": 0.7142857142857143,
   "blunders": 0.07142857142857142,
   "error_ci": [
    1.9982226547740756,
    13.980105874084291
   ],
   "error_positions": 42,
   "error_rate": 7.188437595253899,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "0a97729e+dirty"
   ],
   "invalid": 0,
   "paired": [
    {
     "ci": [
      -13.317110283034188,
      7.497211297353108
     ],
     "diff": -1.713874794188,
     "positions": 42,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      -1.4756974719819569,
      7.871319140706744
     ],
     "diff": 2.7081476790564403,
     "positions": 42,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "none (choice only)",
   "provider": "openrouter:xiaomi/fp8",
   "quantization": "fp8",
   "reasoning": "high",
   "request_s_p50": 23.5765,
   "request_s_p90": 346.14779999999985,
   "retry_s_max": 0.0,
   "retry_s_mean": 0.0,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "MiMo V2.6 Pro (high)",
   "spec": "openai-compatible/xiaomi/mimo-v2.6-pro",
   "timed_out": 8,
   "tokens_in": 3132.095238095238,
   "tokens_out": 4333.309523809524,
   "tokens_reasoning": 4324.285714285715,
   "usd_per_1000": 4.0464631142857135,
   "usd_total": 0.1699514508
  },
  {
   "answered": 50,
   "asked": 50,
   "best_play": 0.64,
   "blunders": 0.16,
   "error_ci": [
    7.30754554271698,
    24.19400066137314
   ],
   "error_positions": 50,
   "error_rate": 15.122810900211334,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "0a97729e+dirty"
   ],
   "invalid": 0,
   "paired": [
    {
     "ci": [
      -9.04761791229248,
      10.629656910896301
     ],
     "diff": 1.5513616800308228,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      1.0293889045715332,
      13.259136080741882
     ],
     "diff": 6.5911126136779785,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "letter logprobs",
   "provider": "openrouter:nebius/fp8",
   "quantization": "fp8",
   "reasoning": "low",
   "request_s_p50": 7.663,
   "request_s_p90": 16.2471,
   "retry_s_max": 0.0,
   "retry_s_mean": 0.0,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "Beam 1.0 (low)",
   "spec": "openai-compatible/reflection/beam-1.0-501b-a23b",
   "timed_out": 0,
   "tokens_in": 3248.02,
   "tokens_out": 1548.2,
   "tokens_reasoning": 1538.68,
   "usd_per_1000": 1.985677,
   "usd_total": 0.09928384999999999
  },
  {
   "answered": 50,
   "asked": 50,
   "best_play": 0.64,
   "blunders": 0.12,
   "error_ci": [
    5.920412540435791,
    26.66627585887909
   ],
   "error_positions": 50,
   "error_rate": 15.019604861736298,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "0a97729e+dirty"
   ],
   "invalid": 0,
   "paired": [
    {
     "ci": [
      -8.878282308578491,
      12.212541103363037
     ],
     "diff": 1.4481556415557861,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      -3.832477331161499,
      18.406342267990112
     ],
     "diff": 6.487906575202942,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "letter logprobs",
   "provider": "openrouter:nebius/fp8",
   "quantization": "fp8",
   "reasoning": "high",
   "request_s_p50": 34.4735,
   "request_s_p90": 72.9444,
   "retry_s_max": 0.0,
   "retry_s_mean": 0.0,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "Beam 1.0 (high)",
   "spec": "openai-compatible/reflection/beam-1.0-501b-a23b",
   "timed_out": 0,
   "tokens_in": 3249.02,
   "tokens_out": 8227.38,
   "tokens_reasoning": 8217.64,
   "usd_per_1000": 8.673555,
   "usd_total": 0.43367775
  },
  {
   "answered": 50,
   "asked": 50,
   "best_play": 0.6222222222222222,
   "blunders": 0.13333333333333333,
   "error_ci": [
    5.682551860809326,
    20.59470083978441
   ],
   "error_positions": 45,
   "error_rate": 12.503284878200954,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "0a97729e+dirty"
   ],
   "invalid": 5,
   "paired": [
    {
     "ci": [
      -10.755023029115465,
      11.095597015486824
     ],
     "diff": 0.8448372284571329,
     "positions": 45,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      0.5800303485658433,
      13.780246178309122
     ],
     "diff": 6.4445604880650835,
     "positions": 45,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "letter logprobs",
   "provider": "openrouter:makora/fp8",
   "quantization": "fp8",
   "reasoning": "off",
   "request_s_p50": 0.854,
   "request_s_p90": 2.7089000000000003,
   "retry_s_max": 0.0,
   "retry_s_mean": 0.0,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "MiMo V2.6 Flash (off)",
   "spec": "openai-compatible/xiaomi/mimo-v2.6-flash",
   "timed_out": 0,
   "tokens_in": 3209.02,
   "tokens_out": 31.32,
   "tokens_reasoning": 0,
   "usd_per_1000": 0.14446508,
   "usd_total": 0.007223254
  },
  {
   "answered": 50,
   "asked": 50,
   "best_play": 0.6041666666666666,
   "blunders": 0.16666666666666666,
   "error_ci": [
    7.850880424181621,
    25.743213792641956
   ],
   "error_positions": 48,
   "error_rate": 15.934731190403303,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "0a97729e+dirty"
   ],
   "invalid": 2,
   "paired": [
    {
     "ci": [
      -7.205717576046784,
      17.0840152228872
     ],
     "diff": 5.301226861774921,
     "positions": 48,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      3.0177303900321326,
      19.2846084634463
     ],
     "diff": 10.1234195753932,
     "positions": 48,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "letter logprobs",
   "provider": "openrouter:makora/fp8",
   "quantization": "fp8",
   "reasoning": "off",
   "request_s_p50": 0.7575000000000001,
   "request_s_p90": 1.5551,
   "retry_s_max": 0.0,
   "retry_s_mean": 0.0,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "MiMo V2.6 Flash (off, unconstrained)",
   "spec": "openai-compatible/xiaomi/mimo-v2.6-flash",
   "timed_out": 0,
   "tokens_in": 3209.02,
   "tokens_out": 2.56,
   "tokens_reasoning": 0,
   "usd_per_1000": 0.0197582,
   "usd_total": 0.00098791
  },
  {
   "answered": 50,
   "asked": 50,
   "best_play": 0.75,
   "blunders": 0.08333333333333333,
   "error_ci": [
    1.5827040705415938,
    15.35409978694386
   ],
   "error_positions": 36,
   "error_rate": 7.72282439801428,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "0a97729e+dirty"
   ],
   "invalid": 14,
   "paired": [
    {
     "ci": [
      -0.15528003374735513,
      11.406980454921722
     ],
     "diff": 4.368414481480916,
     "positions": 36,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      0.10116232766045465,
      9.636287060048845
     ],
     "diff": 3.867821560965644,
     "positions": 36,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "letter logprobs",
   "provider": "openrouter:makora/fp8",
   "quantization": "fp8",
   "reasoning": "low",
   "request_s_p50": 39.45399999999999,
   "request_s_p90": 332.6569,
   "retry_s_max": 0.0,
   "retry_s_mean": 0.0,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "MiMo V2.6 Flash (low)",
   "spec": "openai-compatible/xiaomi/mimo-v2.6-flash",
   "timed_out": 0,
   "tokens_in": 3207.02,
   "tokens_out": 12660.5,
   "tokens_reasoning": 12652.6,
   "usd_per_1000": 3.62466988,
   "usd_total": 0.181233494
  },
  {
   "answered": 50,
   "asked": 50,
   "best_play": 0.6,
   "blunders": 0.14,
   "error_ci": [
    7.228196859359741,
    28.231877088546753
   ],
   "error_positions": 50,
   "error_rate": 16.730687618255615,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "0a97729e+dirty"
   ],
   "invalid": 0,
   "paired": [
    {
     "ci": [
      -1.706235408782959,
      9.863335490226746
     ],
     "diff": 3.1592383980751038,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      0.5782172083854675,
      18.371108770370483
     ],
     "diff": 8.19898933172226,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "none (choice only)",
   "provider": "openrouter:inference-net/fp4",
   "quantization": "fp4",
   "reasoning": "low",
   "request_s_p50": 16.5445,
   "request_s_p90": 42.16410000000002,
   "retry_s_max": 15.193,
   "retry_s_mean": 0.60742,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "Kimi K3 (low)",
   "spec": "openai-compatible/moonshotai/kimi-k3",
   "timed_out": 0,
   "tokens_in": 3231.9,
   "tokens_out": 722.94,
   "tokens_reasoning": 715.54,
   "usd_per_1000": 9.3482801,
   "usd_total": 0.467414005
  },
  {
   "answered": 50,
   "asked": 50,
   "best_play": 0.58,
   "blunders": 0.22,
   "error_ci": [
    10.809550285339355,
    33.52125108242035
   ],
   "error_positions": 50,
   "error_rate": 20.8996120095253,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "0a97729e+dirty"
   ],
   "invalid": 0,
   "paired": [
    {
     "ci": [
      -5.007856488227844,
      20.486416816711426
     ],
     "diff": 7.328162789344788,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      4.068813323974609,
      23.80726456642151
     ],
     "diff": 12.367913722991943,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "none (choice only)",
   "provider": "openrouter:tencent/fp8",
   "quantization": "fp8",
   "reasoning": "off",
   "request_s_p50": 2.706,
   "request_s_p90": 3.9806,
   "retry_s_max": 0.0,
   "retry_s_mean": 0.0,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "Hy4 Preview (off)",
   "spec": "openai-compatible/tencent/hy4-preview",
   "timed_out": 0,
   "tokens_in": 3069.22,
   "tokens_out": 8.4,
   "tokens_reasoning": 0,
   "usd_per_1000": 1.70890428,
   "usd_total": 0.085445214
  },
  {
   "answered": 50,
   "asked": 50,
   "best_play": 0.6,
   "blunders": 0.18,
   "error_ci": [
    9.426678717136383,
    38.32369685173035
   ],
   "error_positions": 50,
   "error_rate": 22.13794618844986,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "0a97729e+dirty"
   ],
   "invalid": 0,
   "paired": [
    {
     "ci": [
      -6.381126642227173,
      25.890142917633057
     ],
     "diff": 8.566496968269348,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      2.3043042421340942,
      29.022024869918823
     ],
     "diff": 13.606247901916504,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "none (choice only)",
   "provider": "openrouter:minimax",
   "quantization": null,
   "reasoning": "low",
   "request_s_p50": 4.2915,
   "request_s_p90": 10.658000000000003,
   "retry_s_max": 11.876,
   "retry_s_mean": 7.0504,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "MiniMax-M3.1-Flash (low)",
   "spec": "openai-compatible/minimax/minimax-m3.1-flash:free",
   "timed_out": 0,
   "tokens_in": 3225.22,
   "tokens_out": 392.64,
   "tokens_reasoning": 391.02,
   "usd_per_1000": 0.719367,
   "usd_total": 0.035968349999999996
  },
  {
   "answered": 50,
   "asked": 50,
   "best_play": 0.78,
   "blunders": 0.08,
   "error_ci": [
    2.834230363368988,
    18.001069128513336
   ],
   "error_positions": 50,
   "error_rate": 9.452761113643646,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "0a97729e+dirty"
   ],
   "invalid": 0,
   "paired": [
    {
     "ci": [
      -14.947074055671692,
      5.030655860900879
     ],
     "diff": -4.118688106536865,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      -2.5546908378601074,
      5.93138575553894
     ],
     "diff": 0.9210628271102905,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "none (choice only)",
   "provider": "xai-subscription",
   "quantization": null,
   "reasoning": "low",
   "request_s_p50": 18.747999999999998,
   "request_s_p90": 64.16430000000001,
   "retry_s_max": 0.0,
   "retry_s_mean": 0.0,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "Grok 4.7 (low)",
   "spec": "xai/grok-4.7",
   "timed_out": 0,
   "tokens_in": 4354.3,
   "tokens_out": 2010.64,
   "tokens_reasoning": 2004.64,
   "usd_per_1000": 20.77244,
   "usd_total": 1.038622
  },
  {
   "answered": 46,
   "asked": 50,
   "best_play": 0.8260869565217391,
   "blunders": 0.043478260869565216,
   "error_ci": [
    0.9476714160131372,
    14.689943388752315
   ],
   "error_positions": 46,
   "error_rate": 6.5481698383455695,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "0a97729e+dirty"
   ],
   "invalid": 0,
   "paired": [
    {
     "ci": [
      -17.625828799994096,
      1.9954637340877368
     ],
     "diff": -6.892039724018263,
     "positions": 46,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      -4.102182777031608,
      0.7304089224856832
     ],
     "diff": -1.4140495787496152,
     "positions": 46,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "none (choice only)",
   "provider": "xai-subscription",
   "quantization": null,
   "reasoning": "high",
   "request_s_p50": 69.17099999999999,
   "request_s_p90": 264.5915,
   "retry_s_max": 0.0,
   "retry_s_mean": 0.0,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "Grok 4.7 (high)",
   "spec": "xai/grok-4.7",
   "timed_out": 4,
   "tokens_in": 4335.847826086957,
   "tokens_out": 9178.173913043478,
   "tokens_reasoning": 9172.173913043478,
   "usd_per_1000": 63.740739130434775,
   "usd_total": 2.932074
  },
  {
   "answered": 50,
   "asked": 50,
   "best_play": 0.56,
   "blunders": 0.26,
   "error_ci": [
    17.76912212371826,
    53.66371154785156
   ],
   "error_positions": 50,
   "error_rate": 34.49785530567169,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "ef186da9+dirty"
   ],
   "invalid": 0,
   "paired": [
    {
     "ci": [
      6.334142088890076,
      39.17684108018875
     ],
     "diff": 20.92640608549118,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      10.107127130031586,
      44.9531415104866
     ],
     "diff": 25.966157019138336,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "decision model",
   "provider": "local (d3_server.py)",
   "quantization": "bf16",
   "reasoning": "off",
   "request_s_p50": 0.6385000000000001,
   "request_s_p90": 0.81,
   "retry_s_max": 0.0,
   "retry_s_mean": 0.0,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "d3-turbo",
   "spec": "d3/d3-turbo",
   "timed_out": 0,
   "tokens_in": 3350.78,
   "tokens_out": 0,
   "tokens_reasoning": null,
   "usd_per_1000": null,
   "usd_total": 0
  },
  {
   "answered": 50,
   "asked": 50,
   "best_play": 0.48,
   "blunders": 0.2,
   "error_ci": [
    12.548425197601318,
    35.01433789730072
   ],
   "error_positions": 50,
   "error_rate": 23.159749507904053,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "ef186da9+dirty"
   ],
   "invalid": 0,
   "paired": [
    {
     "ci": [
      -3.420184552669525,
      22.692365050315857
     ],
     "diff": 9.588300287723541,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      5.840212106704712,
      25.67781686782837
     ],
     "diff": 14.628051221370697,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "letter logprobs",
   "provider": "local-vllm",
   "quantization": "iq2_xxs (NVIDIA's 24GB checkpoint: routed experts IQ2_XXS 2.06 bits, quantization-aware distilled; Q8_0 and W4A16 NVFP4 elsewhere; fp8 KV cache)",
   "reasoning": "off",
   "request_s_p50": 2.718,
   "request_s_p90": 14.011,
   "retry_s_max": 0.0,
   "retry_s_mean": 0.0,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "Nemotron 3.5 Bolt (off)",
   "spec": "local/nemotron-3.5-bolt",
   "timed_out": 0,
   "tokens_in": 3228.02,
   "tokens_out": 1,
   "tokens_reasoning": 0,
   "usd_per_1000": null,
   "usd_total": 0
  },
  {
   "answered": 50,
   "asked": 50,
   "best_play": 0.66,
   "blunders": 0.1,
   "error_ci": [
    5.13763964176178,
    19.83319878578186
   ],
   "error_positions": 50,
   "error_rate": 11.591983139514923,
   "failed": 0,
   "group_sizes": [],
   "harness": [
    "ef186da9+dirty"
   ],
   "invalid": 0,
   "paired": [
    {
     "ci": [
      -11.658774614334106,
      5.637108683586121
     ],
     "diff": -1.9794660806655884,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low) [antigravity]"
    },
    {
     "ci": [
      0.641179084777832,
      6.442545652389526
     ],
     "diff": 3.0602848529815674,
     "positions": 50,
     "reference": "Gemini 3.8 Flash (low)"
    }
   ],
   "probabilities": "letter logprobs",
   "provider": "local-vllm",
   "quantization": "iq2_xxs (NVIDIA's 24GB checkpoint: routed experts IQ2_XXS 2.06 bits, quantization-aware distilled; Q8_0 and W4A16 NVFP4 elsewhere; fp8 KV cache)",
   "reasoning": "template default",
   "request_s_p50": 218.01100000000002,
   "request_s_p90": 303.3743,
   "retry_s_max": 0.0,
   "retry_s_mean": 0.0,
   "run_dates": [
    "2026-10-21"
   ],
   "setup": "Nemotron 3.5 Bolt (on)",
   "spec": "local/nemotron-3.5-bolt",
   "timed_out": 0,
   "tokens_in": 29616.02,
   "tokens_out": 23168.98,
   "tokens_reasoning": 0,
   "usd_per_1000": null,
   "usd_total": 0
  }
 ],
 "status": "Smoke test"
}
