{
 "generated_at": "2026-11-16T22:07:58Z",
 "tests": {
  "logic": {
   "name": "Logic",
   "questions": 1125,
   "setups": [
    {
     "name": "GPT-6 Astra",
     "display_name": "GPT-6 Astra",
     "tag": "chat",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "GPT-6 Astra",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9671111111111111,
     "wilson_low": 0.9549966802141904,
     "wilson_high": 0.9760462579624326,
     "right": 1088,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.5636363636363636,
     "cost_per_1000": 0.0,
     "cost_usd": 0.0,
     "cost_unit": "USD",
     "cost_basis": "ChatGPT subscription",
     "gpu_hours": null,
     "median_latency_ms": 6590.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemini 4 Carbon",
     "display_name": "Gemini 4 Carbon",
     "tag": "chat",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "Gemini 4 Carbon (Antigravity subscription)",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9671111111111111,
     "wilson_low": 0.9549966802141904,
     "wilson_high": 0.9760462579624326,
     "right": 1088,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.45454545454545453,
     "cost_per_1000": null,
     "cost_usd": null,
     "cost_unit": "USD",
     "cost_basis": null,
     "gpu_hours": null,
     "median_latency_ms": 44520.0,
     "mixed_concurrency": false
    },
    {
     "name": "GPT-6.1 Sol",
     "display_name": "GPT-6.1 Sol",
     "tag": "chat",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "GPT-6.1 Sol",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9653333333333334,
     "wilson_low": 0.9529611692413362,
     "wilson_high": 0.9745383134141579,
     "right": 1086,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.631578947368421,
     "cost_per_1000": 0.0,
     "cost_usd": 0.0,
     "cost_unit": "USD",
     "cost_basis": "ChatGPT subscription",
     "gpu_hours": null,
     "median_latency_ms": 7070.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen4-Max",
     "display_name": "Qwen4-Max",
     "tag": "high · chat",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "Qwen4-Max",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9626666666666667,
     "wilson_low": 0.9499213730037847,
     "wilson_high": 0.9722629263700159,
     "right": 1083,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.0,
     "cost_per_1000": 5.689670133333334,
     "cost_usd": 6.4008789,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 11570.0,
     "mixed_concurrency": false
    },
    {
     "name": "Claude Opus 5.5",
     "display_name": "Claude Opus 5.5",
     "tag": "chat",
     "class_label": "Closed · Flagship",
     "base_model": null,
     "setup_label": "Claude Opus 5.5",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9617777777777777,
     "wilson_low": 0.9489114556883422,
     "wilson_high": 0.9715011159248939,
     "right": 1082,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.38181818181818183,
     "cost_per_1000": 19.54015288888889,
     "cost_usd": 21.982672,
     "cost_unit": "USD",
     "cost_basis": "Claude subscription",
     "gpu_hours": null,
     "median_latency_ms": 5830.0,
     "mixed_concurrency": false
    },
    {
     "name": "DeepSeek-V4.1-Flash",
     "display_name": "DeepSeek-V4.1-Flash",
     "tag": "high · chat · 1017",
     "class_label": "Open · 552B MoE · 16B active",
     "base_model": null,
     "setup_label": "DeepSeek-V4.1-Flash 1017",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9617777777777777,
     "wilson_low": 0.9489114556883422,
     "wilson_high": 0.9715011159248939,
     "right": 1082,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.03773584905660377,
     "cost_per_1000": 1.646917016,
     "cost_usd": 1.852781643,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 6680.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemini 4 Hydrogen",
     "display_name": "Gemini 4 Hydrogen",
     "tag": "chat",
     "class_label": "",
     "base_model": null,
     "setup_label": "Gemini 4 Hydrogen (Antigravity subscription)",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9608888888888889,
     "wilson_low": 0.9479031208614758,
     "wilson_high": 0.9707377229911959,
     "right": 1081,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.7605633802816901,
     "cost_per_1000": null,
     "cost_usd": null,
     "cost_unit": "USD",
     "cost_basis": null,
     "gpu_hours": null,
     "median_latency_ms": 8210.0,
     "mixed_concurrency": false
    },
    {
     "name": "Inkling 2 Small",
     "display_name": "Inkling 2 Small",
     "tag": "high · chat",
     "class_label": "Open · 594B MoE · 17B active",
     "base_model": null,
     "setup_label": "Inkling 2 Small",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9564444444444444,
     "wilson_low": 0.9428835110822366,
     "wilson_high": 0.9668986939676129,
     "right": 1076,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.3728813559322034,
     "cost_per_1000": 4.121375804444444,
     "cost_usd": 4.63654778,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 13180.0,
     "mixed_concurrency": false
    },
    {
     "name": "Claude Sonnet 5.5",
     "display_name": "Claude Sonnet 5.5",
     "tag": "chat",
     "class_label": "Closed · Workhorse",
     "base_model": null,
     "setup_label": "Claude Sonnet 5.5",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9546666666666667,
     "wilson_low": 0.9408851714281397,
     "wilson_high": 0.9653535781005809,
     "right": 1074,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.4126984126984127,
     "cost_per_1000": 11.479900977777778,
     "cost_usd": 12.9148886,
     "cost_unit": "USD",
     "cost_basis": "Claude subscription",
     "gpu_hours": null,
     "median_latency_ms": 6710.0,
     "mixed_concurrency": false
    },
    {
     "name": "Muse Ember",
     "display_name": "Muse Ember",
     "tag": "max · chat",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Muse Ember",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9546666666666667,
     "wilson_low": 0.9408851714281397,
     "wilson_high": 0.9653535781005809,
     "right": 1074,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.22641509433962265,
     "cost_per_1000": 1.8204486364444443,
     "cost_usd": 2.048004716,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 3110.0,
     "mixed_concurrency": false
    },
    {
     "name": "Inkling-Small",
     "display_name": "Inkling-Small",
     "tag": "high · chat · Jul 2026",
     "class_label": "Open · 276B MoE · 12B active",
     "base_model": null,
     "setup_label": "Inkling Small",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9537777777777777,
     "wilson_low": 0.9398878869005304,
     "wilson_high": 0.9645791348676255,
     "right": 1073,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.5,
     "cost_per_1000": 5.489383466666666,
     "cost_usd": 6.1755564,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 13290.0,
     "mixed_concurrency": false
    },
    {
     "name": "Claude Fable 5.1",
     "display_name": "Claude Fable 5.1",
     "tag": "chat",
     "class_label": "Closed · Flagship",
     "base_model": null,
     "setup_label": "Claude Fable 5.1",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.952,
     "wilson_low": 0.9378969212615481,
     "wilson_high": 0.9630266449854789,
     "right": 1071,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.08333333333333333,
     "cost_per_1000": 0.0,
     "cost_usd": 0.0,
     "cost_unit": "USD",
     "cost_basis": "Claude subscription",
     "gpu_hours": null,
     "median_latency_ms": 3820.0,
     "mixed_concurrency": false
    },
    {
     "name": "MiMo-V2.6-Flash",
     "display_name": "MiMo-V2.6-Flash",
     "tag": "on · chat",
     "class_label": "Open · 309B MoE · 15B active",
     "base_model": null,
     "setup_label": "MiMo V2.6 Flash",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9511111111111111,
     "wilson_low": 0.9369031778110767,
     "wilson_high": 0.9622486606753861,
     "right": 1070,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.2328767123287671,
     "cost_per_1000": 1.3901424419555555,
     "cost_usd": 1.5639102472,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 24740.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen3.5-397B-A17B",
     "display_name": "Qwen3.5-397B-A17B",
     "tag": "on · chat",
     "class_label": "Open · 397B MoE · 17B active",
     "base_model": null,
     "setup_label": "Qwen3.5-397B-A17B",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9511111111111111,
     "wilson_low": 0.9369031778110767,
     "wilson_high": 0.9622486606753861,
     "right": 1070,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.31746031746031744,
     "cost_per_1000": 13.425631093333333,
     "cost_usd": 15.10383498,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 84650.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen3.8-27B",
     "display_name": "Qwen3.8-27B",
     "tag": "low · chat",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Qwen3.8-27B (chat, reasoning low, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9404444444444444,
     "wilson_low": 0.925058443671722,
     "wilson_high": 0.9528326616879672,
     "right": 1058,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.391304347826087,
     "cost_per_1000": 4.884088888888889,
     "cost_usd": 5.4946,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 2.2066666666666666,
     "median_latency_ms": 18710.0,
     "mixed_concurrency": false
    },
    {
     "name": "GPT-6.1 Terra",
     "display_name": "GPT-6.1 Terra",
     "tag": "chat",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "GPT-6.1 Terra",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9395555555555556,
     "wilson_low": 0.924077387665377,
     "wilson_high": 0.9520419899337478,
     "right": 1057,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.6111111111111112,
     "cost_per_1000": 0.0,
     "cost_usd": 0.0,
     "cost_unit": "USD",
     "cost_basis": "ChatGPT subscription",
     "gpu_hours": null,
     "median_latency_ms": 5640.0,
     "mixed_concurrency": false
    },
    {
     "name": "MiniMax-M3.1-Flash",
     "display_name": "MiniMax-M3.1-Flash",
     "tag": "high · chat",
     "class_label": "Open · 428B MoE · 23B active",
     "base_model": null,
     "setup_label": "MiniMax-M3.1-Flash",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9395555555555556,
     "wilson_low": 0.924077387665377,
     "wilson_high": 0.9520419899337478,
     "right": 1057,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.3829787234042553,
     "cost_per_1000": 0.8032300333333333,
     "cost_usd": 0.9036337875,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 5610.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen3.8 Flash",
     "display_name": "Qwen3.8 Flash",
     "tag": "off · chat",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "Qwen3.8 Flash",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9395555555555556,
     "wilson_low": 0.924077387665377,
     "wilson_high": 0.9520419899337478,
     "right": 1057,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.345679012345679,
     "cost_per_1000": 1.3762621546666665,
     "cost_usd": 1.548294924,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 17550.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 31B",
     "display_name": "Gemma 4 31B",
     "tag": "chat",
     "class_label": "Open · 31B",
     "base_model": null,
     "setup_label": "Gemma 4 31B (chat, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9333333333333333,
     "wilson_low": 0.9172324555980313,
     "wilson_high": 0.9464848276771423,
     "right": 1050,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.16326530612244897,
     "cost_per_1000": 1.2437703703703704,
     "cost_usd": 1.3992416666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.5619444444444445,
     "median_latency_ms": 18290.0,
     "mixed_concurrency": false
    },
    {
     "name": "GLM-5.3-Flash",
     "display_name": "GLM-5.3-Flash",
     "tag": "high · chat",
     "class_label": "Open · 320B MoE · 18B active",
     "base_model": null,
     "setup_label": "GLM-5.3-Flash",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9315555555555556,
     "wilson_low": 0.9152835323879337,
     "wilson_high": 0.944890295366111,
     "right": 1048,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.04225352112676056,
     "cost_per_1000": 1.0565955911111113,
     "cost_usd": 1.18867004,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 12050.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen3.5-4B",
     "display_name": "Qwen3.5-4B",
     "tag": "on · chat",
     "class_label": "Open · 4.7B",
     "base_model": null,
     "setup_label": "Qwen3.5-4B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9235555555555556,
     "wilson_low": 0.9065468224488583,
     "wilson_high": 0.9376814554601062,
     "right": 1039,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.36082474226804123,
     "cost_per_1000": 3.5247333333333337,
     "cost_usd": 3.9653250000000004,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 1.5925,
     "median_latency_ms": 142520.0,
     "mixed_concurrency": false
    },
    {
     "name": "Granite-4.2-3B",
     "display_name": "Granite-4.2-3B",
     "tag": "high · chat",
     "class_label": "Open · 3.0B",
     "base_model": "Granite-4.1-3B-Base",
     "setup_label": "Granite-4.2-3B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9164444444444444,
     "wilson_low": 0.8988222835462485,
     "wilson_high": 0.9312321722782002,
     "right": 1031,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.532258064516129,
     "cost_per_1000": 10.892674074074076,
     "cost_usd": 12.254258333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 4.921388888888889,
     "median_latency_ms": 221350.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen3.5-4B",
     "display_name": "Qwen3.5-4B",
     "tag": "off · chat",
     "class_label": "Open · 4.7B",
     "base_model": null,
     "setup_label": "Qwen3.5-4B (chat, thinking off, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9013333333333333,
     "wilson_low": 0.8825172855828651,
     "wilson_high": 0.9174177983119878,
     "right": 1014,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.3357664233576642,
     "cost_per_1000": 1.4202222222222225,
     "cost_usd": 1.5977500000000002,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.6416666666666667,
     "median_latency_ms": 45380.0,
     "mixed_concurrency": false
    },
    {
     "name": "MiniCPM5-2B",
     "display_name": "MiniCPM5-2B",
     "tag": "on · chat",
     "class_label": "Open · 2.5B",
     "base_model": null,
     "setup_label": "MiniCPM5-2B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.8885925925925925,
     "wilson_low": null,
     "wilson_high": null,
     "right": null,
     "n": null,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.5151515151515151,
     "cost_per_1000": 3.3345506172839507,
     "cost_usd": 3.8283750000000003,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 1.5375,
     "median_latency_ms": 101100.0,
     "mixed_concurrency": false,
     "seed_runs": [
      {
       "seed": 0,
       "run_id": "minicpm5-2b_en",
       "right": 998,
       "n": 1125
      },
      {
       "seed": 1,
       "run_id": "minicpm5-2b-seed1_en",
       "right": 993,
       "n": 1125
      },
      {
       "seed": 2,
       "run_id": "minicpm5-2b-seed2_en",
       "right": 1008,
       "n": 1125
      }
     ],
     "seeds_complete": "3/3",
     "accuracy_basis": "mean of 3 seeds",
     "seed_min": 0.8826666666666667,
     "seed_max": 0.896
    },
    {
     "name": "Gemma 4 12B Unified",
     "display_name": "Gemma 4 12B Unified",
     "tag": "chat",
     "class_label": "Open · 12B",
     "base_model": null,
     "setup_label": "Gemma 4 12B (chat, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.8773333333333333,
     "wilson_low": 0.856868685479114,
     "wilson_high": 0.8952297488804984,
     "right": 987,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.15492957746478872,
     "cost_per_1000": 1.2917259259259262,
     "cost_usd": 1.4531916666666669,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.5836111111111111,
     "median_latency_ms": 42060.0,
     "mixed_concurrency": false
    },
    {
     "name": "LFM2.5-24B-A2B",
     "display_name": "LFM2.5-24B-A2B",
     "tag": "on · chat",
     "class_label": "Open · 24B MoE · 2.5B active",
     "base_model": null,
     "setup_label": "LFM2.5-24B-A2B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.8666666666666667,
     "wilson_low": 0.845549070452537,
     "wilson_high": 0.8852886307803021,
     "right": 975,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.3782051282051282,
     "cost_per_1000": 7.561607407407409,
     "cost_usd": 8.506808333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 3.4163888888888887,
     "median_latency_ms": 228980.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 E4B",
     "display_name": "Gemma 4 E4B",
     "tag": "on · chat · 32k cap · sampled",
     "class_label": "Open · 8.0B",
     "base_model": null,
     "setup_label": "Gemma 4 E4B (chat, thinking on, 32k, sampled, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.858962962962963,
     "wilson_low": null,
     "wilson_high": null,
     "right": null,
     "n": null,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.5592417061611374,
     "cost_per_1000": 0.9925160493827162,
     "cost_usd": 1.09145,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.43833333333333335,
     "median_latency_ms": 31566.666666666668,
     "mixed_concurrency": false,
     "seed_runs": [
      {
       "seed": 0,
       "run_id": "gemma-4-e4b-it-think-32k-sampled-seed0_en",
       "right": 968,
       "n": 1125
      },
      {
       "seed": 1,
       "run_id": "gemma-4-e4b-it-think-32k-sampled-seed1_en",
       "right": 969,
       "n": 1125
      },
      {
       "seed": 2,
       "run_id": "gemma-4-e4b-it-think-32k-sampled-seed2_en",
       "right": 962,
       "n": 1125
      }
     ],
     "seeds_complete": "3/3",
     "accuracy_basis": "mean of 3 seeds",
     "seed_min": 0.8551111111111112,
     "seed_max": 0.8613333333333333
    },
    {
     "name": "LFM3-1.7B",
     "display_name": "LFM3-1.7B",
     "tag": "off · chat",
     "class_label": "Open · 1.7B",
     "base_model": null,
     "setup_label": "LFM3-1.7B (chat, thinking off, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.8551111111111112,
     "wilson_low": 0.8333333333333335,
     "wilson_high": 0.8744719070121678,
     "right": 962,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.5416666666666666,
     "cost_per_1000": 0.4867485242029732,
     "cost_usd": 0.5475920897283448,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 2.5730555555555554,
     "median_latency_ms": 163870.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 E4B",
     "display_name": "Gemma 4 E4B",
     "tag": "on · chat · 32k cap",
     "class_label": "Open · 8.0B",
     "base_model": null,
     "setup_label": "Gemma 4 E4B (chat, thinking on, 32k, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.8497777777777777,
     "wilson_low": 0.8277105168142238,
     "wilson_high": 0.8694643569678906,
     "right": 956,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": null,
     "cost_per_1000": 1.5026074074074074,
     "cost_usd": 1.6904333333333335,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.6788888888888889,
     "median_latency_ms": 57900.0,
     "mixed_concurrency": false
    },
    {
     "name": "LFM3-1.7B",
     "display_name": "LFM3-1.7B",
     "tag": "off · chat · NVIDIA RTX PRO 6000 Blackwell",
     "class_label": "Open · 1.7B",
     "base_model": null,
     "setup_label": "LFM3-1.7B (chat, thinking off, RTX PRO 6000, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.8456296296296296,
     "wilson_low": null,
     "wilson_high": null,
     "right": null,
     "n": null,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": null,
     "cost_per_1000": 1.039651851851852,
     "cost_usd": 1.1806750000000001,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.4741666666666667,
     "median_latency_ms": 27950.0,
     "mixed_concurrency": false,
     "seed_runs": [
      {
       "seed": 0,
       "run_id": "lfm3-1.7b-vllm-nothink-rtx-seed0_en",
       "right": 956,
       "n": 1125
      },
      {
       "seed": 1,
       "run_id": "lfm3-1.7b-vllm-nothink-rtx-seed1_en",
       "right": 949,
       "n": 1125
      },
      {
       "seed": 2,
       "run_id": "lfm3-1.7b-vllm-nothink-rtx-seed2_en",
       "right": 949,
       "n": 1125
      }
     ],
     "seeds_complete": "3/3",
     "accuracy_basis": "mean of 3 seeds",
     "seed_min": 0.8435555555555555,
     "seed_max": 0.8497777777777777
    },
    {
     "name": "LFM3-1.7B",
     "display_name": "LFM3-1.7B",
     "tag": "on · chat",
     "class_label": "Open · 1.7B",
     "base_model": null,
     "setup_label": "LFM3-1.7B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.8423703703703703,
     "wilson_low": null,
     "wilson_high": null,
     "right": null,
     "n": null,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.43612334801762115,
     "cost_per_1000": 1.0173135802469135,
     "cost_usd": 1.1274166666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.4527777777777778,
     "median_latency_ms": 29846.666666666668,
     "mixed_concurrency": false,
     "seed_runs": [
      {
       "seed": 0,
       "run_id": "lfm3-1.7b-vllm_en",
       "right": 950,
       "n": 1125
      },
      {
       "seed": 1,
       "run_id": "lfm3-1.7b-vllm-seed1_en",
       "right": 944,
       "n": 1125
      },
      {
       "seed": 2,
       "run_id": "lfm3-1.7b-vllm-seed2_en",
       "right": 949,
       "n": 1125
      }
     ],
     "seeds_complete": "3/3",
     "accuracy_basis": "mean of 3 seeds",
     "seed_min": 0.8391111111111111,
     "seed_max": 0.8444444444444444
    },
    {
     "name": "MiniCPM5-2B",
     "display_name": "MiniCPM5-2B",
     "tag": "off · chat",
     "class_label": "Open · 2.5B",
     "base_model": null,
     "setup_label": "MiniCPM5-2B (chat, thinking off, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.835851851851852,
     "wilson_low": null,
     "wilson_high": null,
     "right": null,
     "n": null,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.4389438943894389,
     "cost_per_1000": 1.3269753086419753,
     "cost_usd": 1.5942916666666669,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.6402777777777777,
     "median_latency_ms": 46453.333333333336,
     "mixed_concurrency": false,
     "seed_runs": [
      {
       "seed": 0,
       "run_id": "minicpm5-2b-nothink_en",
       "right": 942,
       "n": 1125
      },
      {
       "seed": 1,
       "run_id": "minicpm5-2b-nothink-seed1_en",
       "right": 944,
       "n": 1125
      },
      {
       "seed": 2,
       "run_id": "minicpm5-2b-nothink-seed2_en",
       "right": 935,
       "n": 1125
      }
     ],
     "seeds_complete": "3/3",
     "accuracy_basis": "mean of 3 seeds",
     "seed_min": 0.8311111111111111,
     "seed_max": 0.8391111111111111
    },
    {
     "name": "Gemma 4 E4B",
     "display_name": "Gemma 4 E4B",
     "tag": "chat",
     "class_label": "Open · 8.0B",
     "base_model": null,
     "setup_label": "Gemma 4 E4B (chat, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.832,
     "wilson_low": 0.8090309034963439,
     "wilson_high": 0.8527094150744812,
     "right": 936,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.5238095238095238,
     "cost_per_1000": 0.8422962962962963,
     "cost_usd": 0.9475833333333333,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.38055555555555554,
     "median_latency_ms": 32689.999999999996,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 E4B",
     "display_name": "Gemma 4 E4B",
     "tag": "on · chat",
     "class_label": "Open · 8.0B",
     "base_model": null,
     "setup_label": "Gemma 4 E4B (chat, thinking on, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.8257777777777778,
     "wilson_low": 0.8025143343771703,
     "wilson_high": 0.8468238898697038,
     "right": 929,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.45064377682403434,
     "cost_per_1000": 0.6495410242548798,
     "cost_usd": 0.7307336522867398,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 3.433611111111111,
     "median_latency_ms": 284080.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 E2B",
     "display_name": "Gemma 4 E2B",
     "tag": "on · chat · 32k cap · sampled",
     "class_label": "Open · 5.1B",
     "base_model": null,
     "setup_label": "Gemma 4 E2B (chat, thinking on, 32k, sampled, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.7677037037037037,
     "wilson_low": null,
     "wilson_high": null,
     "right": null,
     "n": null,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.3256578947368421,
     "cost_per_1000": 0.5740320987654322,
     "cost_usd": 0.6425583333333333,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.25805555555555554,
     "median_latency_ms": 21223.333333333332,
     "mixed_concurrency": false,
     "seed_runs": [
      {
       "seed": 0,
       "run_id": "gemma-4-e2b-it-think-32k-sampled-seed0_en",
       "right": 855,
       "n": 1125
      },
      {
       "seed": 1,
       "run_id": "gemma-4-e2b-it-think-32k-sampled-seed1_en",
       "right": 873,
       "n": 1125
      },
      {
       "seed": 2,
       "run_id": "gemma-4-e2b-it-think-32k-sampled-seed2_en",
       "right": 863,
       "n": 1125
      }
     ],
     "seeds_complete": "3/3",
     "accuracy_basis": "mean of 3 seeds",
     "seed_min": 0.76,
     "seed_max": 0.776
    },
    {
     "name": "Gemma 4 E2B",
     "display_name": "Gemma 4 E2B",
     "tag": "on · chat · 32k cap",
     "class_label": "Open · 5.1B",
     "base_model": null,
     "setup_label": "Gemma 4 E2B (chat, thinking on, 32k, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.7591111111111111,
     "wilson_low": 0.7332677579443825,
     "wilson_high": 0.7831908842601571,
     "right": 854,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": null,
     "cost_per_1000": 1.1927407407407409,
     "cost_usd": 1.3418333333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.5388888888888889,
     "median_latency_ms": 41350.0,
     "mixed_concurrency": false
    },
    {
     "name": "LFM2.5-2.6B",
     "display_name": "LFM2.5-2.6B",
     "tag": "on · chat",
     "class_label": "Open · 2.7B",
     "base_model": null,
     "setup_label": "LFM2.5-2.6B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.7537777777777778,
     "wilson_low": 0.7277674299752128,
     "wilson_high": 0.7780608456659401,
     "right": 848,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.4434782608695652,
     "cost_per_1000": 2.4795481481481483,
     "cost_usd": 2.7894916666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 1.1202777777777777,
     "median_latency_ms": 81330.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 E2B",
     "display_name": "Gemma 4 E2B",
     "tag": "on · chat",
     "class_label": "Open · 5.1B",
     "base_model": null,
     "setup_label": "Gemma 4 E2B (chat, thinking on, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.7457777777777778,
     "wilson_low": 0.7195265823553877,
     "wilson_high": 0.770356143440685,
     "right": 839,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.2635135135135135,
     "cost_per_1000": 0.3632616374625018,
     "cost_usd": 0.4086693421453145,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 1.9202777777777778,
     "median_latency_ms": 173220.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 E2B",
     "display_name": "Gemma 4 E2B",
     "tag": "chat",
     "class_label": "Open · 5.1B",
     "base_model": null,
     "setup_label": "Gemma 4 E2B (chat, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.7288888888888889,
     "wilson_low": 0.7021657827465061,
     "wilson_high": 0.754054115598842,
     "right": 820,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.3395061728395062,
     "cost_per_1000": 0.5133703703703705,
     "cost_usd": 0.5775416666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.23194444444444445,
     "median_latency_ms": 21390.0,
     "mixed_concurrency": false
    },
    {
     "name": "Raon-1 nano",
     "display_name": "Raon-1 nano",
     "tag": "on · chat",
     "class_label": "Open · 3.1B",
     "base_model": "Raon-1 mini preview (MoE-to-Dense pruning and distillation)",
     "setup_label": "Raon-1 nano (chat, thinking on, KRAFTON vLLM fork, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.7093333333333334,
     "wilson_low": 0.6821225609280717,
     "wilson_high": 0.7351193266848584,
     "right": 798,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.10526315789473684,
     "cost_per_1000": 15.33040740740741,
     "cost_usd": 17.246708333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 6.926388888888889,
     "median_latency_ms": 493230.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen3.5-2B",
     "display_name": "Qwen3.5-2B",
     "tag": "on · chat",
     "class_label": "Open · 2.0B",
     "base_model": null,
     "setup_label": "Qwen3.5-2B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.7031111111111111,
     "wilson_low": 0.6757577866430229,
     "wilson_high": 0.729082006645956,
     "right": 791,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.33116883116883117,
     "cost_per_1000": 4.33014074074074,
     "cost_usd": 4.871408333333333,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 1.956388888888889,
     "median_latency_ms": 186330.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen3-14B",
     "display_name": "Qwen3-14B",
     "tag": "off · chat",
     "class_label": "Open · 15B",
     "base_model": null,
     "setup_label": "Qwen3-14B (chat, reasoning off, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.6533333333333333,
     "wilson_low": 0.625043831601746,
     "wilson_high": 0.680579207095623,
     "right": 735,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.16063348416289594,
     "cost_per_1000": 1.7718962962962963,
     "cost_usd": 1.9933833333333335,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.8005555555555556,
     "median_latency_ms": 16560.0,
     "mixed_concurrency": false
    },
    {
     "name": "Raon-1 nano",
     "display_name": "Raon-1 nano",
     "tag": "off · chat",
     "class_label": "Open · 3.1B",
     "base_model": "Raon-1 mini preview (MoE-to-Dense pruning and distillation)",
     "setup_label": "Raon-1 nano (chat, thinking off, KRAFTON vLLM fork, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.6506666666666666,
     "wilson_low": 0.6223367764865844,
     "wilson_high": 0.6779710789290911,
     "right": 732,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.3573200992555831,
     "cost_per_1000": 9.270177777777779,
     "cost_usd": 10.42895,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 4.1883333333333335,
     "median_latency_ms": 276720.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen3.5-2B",
     "display_name": "Qwen3.5-2B",
     "tag": "off · chat",
     "class_label": "Open · 2.0B",
     "base_model": null,
     "setup_label": "Qwen3.5-2B (chat, thinking off, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.648,
     "wilson_low": 0.6196306746555845,
     "wilson_high": 0.6753619974783979,
     "right": 729,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.3518930957683742,
     "cost_per_1000": 1.604666666666667,
     "cost_usd": 1.8052500000000002,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.725,
     "median_latency_ms": 71290.0,
     "mixed_concurrency": false
    },
    {
     "name": "Karpo Decision 31B",
     "display_name": "Karpo Decision 31B",
     "tag": "",
     "class_label": "Open · 31B",
     "base_model": null,
     "setup_label": "karpo-decision-31B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.6071111111111112,
     "wilson_low": 0.5782532107839534,
     "wilson_high": 0.6352399843640638,
     "right": 683,
     "n": 1125,
     "decision_score": 71.80244394244617,
     "chosen_answer_brier": 0.17106223226218772,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.8155330157741664,
     "non_hallucination": 0.06564551422319474,
     "cost_per_1000": 0.1284962962962963,
     "cost_usd": 0.14455833333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.058055555555555555,
     "median_latency_ms": 172.93218099803198,
     "mixed_concurrency": false
    },
    {
     "name": "LFM2.5-8B-A1B",
     "display_name": "LFM2.5-8B-A1B",
     "tag": "on · chat",
     "class_label": "Open · 8.3B MoE · 1.5B active",
     "base_model": null,
     "setup_label": "LFM2.5-8B-A1B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.6017777777777777,
     "wilson_low": 0.572871800631879,
     "wilson_high": 0.6299910279527513,
     "right": 677,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.5826086956521739,
     "cost_per_1000": 4.663985185185187,
     "cost_usd": 5.246983333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 2.107222222222222,
     "median_latency_ms": 138140.0,
     "mixed_concurrency": true
    },
    {
     "name": "REAX 1.3",
     "display_name": "REAX 1.3",
     "tag": "",
     "class_label": "Open · 57B",
     "base_model": null,
     "setup_label": "REAX 1.3",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5928888888888889,
     "wilson_low": 0.5639106022314939,
     "wilson_high": 0.621234948747492,
     "right": 667,
     "n": 1125,
     "decision_score": 71.07327437407314,
     "chosen_answer_brier": 0.17142340140742623,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.8162076167156597,
     "non_hallucination": 0.05319148936170213,
     "cost_per_1000": 0.16046666666666667,
     "cost_usd": 0.18052500000000002,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.0725,
     "median_latency_ms": 215.45326299997214,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-31B-Gen1.5",
     "display_name": "KnowLine-31B-Gen1.5",
     "tag": "",
     "class_label": "Open · 31B",
     "base_model": null,
     "setup_label": "KnowLine 31B Gen1.5",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.5902222222222222,
     "wilson_low": 0.5612241354395278,
     "wilson_high": 0.6186062322577648,
     "right": 664,
     "n": 1125,
     "decision_score": 70.72438010965274,
     "chosen_answer_brier": 0.1757346200291672,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.8039457177952591,
     "non_hallucination": 0.004273504273504274,
     "cost_per_1000": 0.06142815770196217,
     "cost_usd": 0.06910667741470745,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.32472222222222225,
     "median_latency_ms": 983.2182690151967,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-31B-Gen2",
     "display_name": "KnowLine-31B-Gen2",
     "tag": "",
     "class_label": "Open · 31B",
     "base_model": "Gemma 4 31B",
     "setup_label": "KnowLine-31B-Gen2",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5893333333333334,
     "wilson_low": 0.5603288397027214,
     "wilson_high": 0.6177298002340067,
     "right": 663,
     "n": 1125,
     "decision_score": 71.0919034526086,
     "chosen_answer_brier": 0.16749526428116132,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.8251976781388546,
     "non_hallucination": 0.0,
     "cost_per_1000": 0.11189629629629631,
     "cost_usd": 0.12588333333333335,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.050555555555555555,
     "median_latency_ms": 150.54753099684604,
     "mixed_concurrency": false
    },
    {
     "name": "Copernicus-31B",
     "display_name": "Copernicus-31B",
     "tag": "",
     "class_label": "Open · 31B",
     "base_model": null,
     "setup_label": "Copernicus-31B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5866666666666667,
     "wilson_low": 0.5576435307418581,
     "wilson_high": 0.6150999259131766,
     "right": 660,
     "n": 1125,
     "decision_score": 70.47599827939513,
     "chosen_answer_brier": 0.1771467010787643,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.801326164874552,
     "non_hallucination": 0.019148936170212766,
     "cost_per_1000": 0.1358740740740741,
     "cost_usd": 0.15285833333333335,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.06138888888888889,
     "median_latency_ms": 184.7244579985272,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE 27B",
     "display_name": "GLiDE 27B",
     "tag": "",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "GLiDE 27B (open)",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.584,
     "wilson_low": 0.5549590872146684,
     "wilson_high": 0.6124691861586729,
     "right": 657,
     "n": 1125,
     "decision_score": 70.66570043259122,
     "chosen_answer_brier": 0.1706859913481757,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.8220413951007558,
     "non_hallucination": 0.44299065420560746,
     "cost_per_1000": 0.03315754278010105,
     "cost_usd": 0.037302235627613686,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.17527777777777778,
     "median_latency_ms": 543.5380580020137,
     "mixed_concurrency": false
    },
    {
     "name": "GPT-6 Luna",
     "display_name": "GPT-6 Luna",
     "tag": "chat",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "GPT-6 Luna (low)",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.5795555555555556,
     "wilson_low": 0.550486931315444,
     "wilson_high": 0.6080827032550752,
     "right": 652,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.5505226480836237,
     "cost_per_1000": 0.0,
     "cost_usd": 0.0,
     "cost_unit": "USD",
     "cost_basis": "ChatGPT subscription",
     "gpu_hours": null,
     "median_latency_ms": 3860.0,
     "mixed_concurrency": false
    },
    {
     "name": "Sieve 31B v3",
     "display_name": "Sieve 31B v3",
     "tag": "",
     "class_label": "Open · 31B",
     "base_model": null,
     "setup_label": "Sieve 31B v3",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.5786666666666667,
     "wilson_low": 0.5495927868586634,
     "wilson_high": 0.6072051199512911,
     "right": 651,
     "n": 1125,
     "decision_score": 70.02878597377777,
     "chosen_answer_brier": 0.1780909471911111,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7966808609928251,
     "non_hallucination": 0.055327868852459015,
     "cost_per_1000": 0.06295203843195098,
     "cost_usd": 0.07082104323594485,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.3327777777777778,
     "median_latency_ms": 1049.5701860054396,
     "mixed_concurrency": false
    },
    {
     "name": "Logits Sigma-1 Duo",
     "display_name": "Logits Sigma-1 Duo",
     "tag": "",
     "class_label": "Open · 57B",
     "base_model": null,
     "setup_label": "Logits Sigma-1 Duo",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5777777777777777,
     "wilson_low": 0.5486987377817577,
     "wilson_high": 0.6063274412676324,
     "right": 650,
     "n": 1125,
     "decision_score": 69.89947294277808,
     "chosen_answer_brier": 0.17978831892221636,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7959935222672064,
     "non_hallucination": 0.07053941908713693,
     "cost_per_1000": 0.21149629629629632,
     "cost_usd": 0.23793333333333336,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.09555555555555556,
     "median_latency_ms": 284.9627090035938,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE",
     "display_name": "GLiDE",
     "tag": "v1.2",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Fastino GLiDE v1.2",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.552,
     "wilson_low": 0.5228124715013982,
     "wilson_high": 0.5808336024916227,
     "right": 621,
     "n": 1125,
     "decision_score": 68.39106923597858,
     "chosen_answer_brier": 0.18417861528042836,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7933121181913453,
     "non_hallucination": 0.5016611295681063,
     "cost_per_1000": 0.013875804444444445,
     "cost_usd": 0.01561028,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.036944444444444446,
     "median_latency_ms": 110.86534895002842,
     "mixed_concurrency": false
    },
    {
     "name": "d1-ultra",
     "display_name": "d1-ultra",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Liquid d1-ultra",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.5493333333333333,
     "wilson_low": 0.5201390886529863,
     "wilson_high": 0.5781918020583411,
     "right": 618,
     "n": 1125,
     "decision_score": 68.08642977463506,
     "chosen_answer_brier": 0.18760473784063184,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7828810887063314,
     "non_hallucination": 0.9102244389027432,
     "cost_per_1000": 0.012642970666666668,
     "cost_usd": 0.014223342,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 181.98416708037257,
     "mixed_concurrency": false
    },
    {
     "name": "Drex Max v1.1",
     "display_name": "Drex Max v1.1",
     "tag": "",
     "class_label": "Closed · Efficient (provisional)",
     "base_model": null,
     "setup_label": "Drex Max 1.1",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.5475555555555556,
     "wilson_low": 0.5183572996337008,
     "wilson_high": 0.5764301355564977,
     "right": 616,
     "n": 1125,
     "decision_score": 68.07234282209284,
     "chosen_answer_brier": 0.18610869911369882,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7934707728420891,
     "non_hallucination": 0.540625,
     "cost_per_1000": 0.022373706666666666,
     "cost_usd": 0.02517042,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 328.7586009828374,
     "mixed_concurrency": false
    },
    {
     "name": "Perplexity Decider 27B v1.3",
     "display_name": "Perplexity Decider 27B v1.3",
     "tag": "NVIDIA RTX PRO 6000 Blackwell",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Perplexity Decider 27B v1.3 (local GPU)",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5475555555555556,
     "wilson_low": 0.5183572996337008,
     "wilson_high": 0.5764301355564977,
     "right": 616,
     "n": 1125,
     "decision_score": 67.97524733466334,
     "chosen_answer_brier": 0.18805060886228872,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.8028665833184497,
     "non_hallucination": 0.18149466192170818,
     "cost_per_1000": 0.08115555555555556,
     "cost_usd": 0.0913,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.03666666666666667,
     "median_latency_ms": 107.62363800131425,
     "mixed_concurrency": false
    },
    {
     "name": "d3-ultra",
     "display_name": "d3-ultra",
     "tag": "",
     "class_label": "Open · 26B",
     "base_model": null,
     "setup_label": "vLLM-SR d3-ultra",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5457777777777778,
     "wilson_low": 0.516575883130536,
     "wilson_high": 0.5746680965385336,
     "right": 614,
     "n": 1125,
     "decision_score": 66.45734467357413,
     "chosen_answer_brier": 0.2166308843062953,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7775964609216137,
     "non_hallucination": 0.10114503816793893,
     "cost_per_1000": 0.10943703703703704,
     "cost_usd": 0.12311666666666668,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.049444444444444444,
     "median_latency_ms": 156.27896995283663,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE",
     "display_name": "GLiDE",
     "tag": "API",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Fastino GLiDE",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.5448888888888889,
     "wilson_low": 0.5156853144554662,
     "wilson_high": 0.573786937453039,
     "right": 613,
     "n": 1125,
     "decision_score": 68.27217954965379,
     "chosen_answer_brier": 0.1794452978958133,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.8141854863376835,
     "non_hallucination": 0.6,
     "cost_per_1000": 0.05203426666666666,
     "cost_usd": 0.058538549999999995,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 270.6908759428188,
     "mixed_concurrency": false
    },
    {
     "name": "Mammoth-1",
     "display_name": "Mammoth-1",
     "tag": "",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Mammoth-1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5448888888888889,
     "wilson_low": 0.5156853144554662,
     "wilson_high": 0.573786937453039,
     "right": 613,
     "n": 1125,
     "decision_score": 68.07069534417319,
     "chosen_answer_brier": 0.1834749820054249,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7949776330546493,
     "non_hallucination": 0.7227272727272728,
     "cost_per_1000": 0.038389511111111115,
     "cost_usd": 0.0431882,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.06805555555555555,
     "median_latency_ms": 221.3518680073321,
     "mixed_concurrency": false
    },
    {
     "name": "d3",
     "display_name": "d3",
     "tag": "",
     "class_label": "Open · 26B",
     "base_model": null,
     "setup_label": "vLLM-SR d3",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5448888888888889,
     "wilson_low": 0.5156853144554662,
     "wilson_high": 0.573786937453039,
     "right": 613,
     "n": 1125,
     "decision_score": 68.33999882926713,
     "chosen_answer_brier": 0.17808891230354623,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.8056943311582382,
     "non_hallucination": 0.14835164835164835,
     "cost_per_1000": 0.05717777777777778,
     "cost_usd": 0.06432500000000001,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.025833333333333333,
     "median_latency_ms": 75.55313900002147,
     "mixed_concurrency": false
    },
    {
     "name": "Erk-14B",
     "display_name": "Erk-14B",
     "tag": "off · chat",
     "class_label": "Open · 15B",
     "base_model": null,
     "setup_label": "Erk-14B (chat, reasoning off, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5422222222222223,
     "wilson_low": 0.5130141661967325,
     "wilson_high": 0.5711429024300794,
     "right": 610,
     "n": 1125,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.3069139966273187,
     "cost_per_1000": 0.3897925925925926,
     "cost_usd": 0.43851666666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.1761111111111111,
     "median_latency_ms": 10150.0,
     "mixed_concurrency": false
    },
    {
     "name": "Perplexity Decider 27B v1.2",
     "display_name": "Perplexity Decider 27B v1.2",
     "tag": "API",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Perplexity Decider 27B v1.2 (API)",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.5333333333333333,
     "wilson_low": 0.5041163686108355,
     "wilson_high": 0.5623234224103316,
     "right": 600,
     "n": 1125,
     "decision_score": 67.64358400703273,
     "chosen_answer_brier": 0.1804616531926787,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7976285714285715,
     "non_hallucination": 0.1436265709156194,
     "cost_per_1000": 0.007457902222222221,
     "cost_usd": 0.008390139999999999,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 237.5898030004464,
     "mixed_concurrency": false
    },
    {
     "name": "Drex Max",
     "display_name": "Drex Max",
     "tag": "",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Drex Max v1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5306666666666666,
     "wilson_low": 0.5014488344249834,
     "wilson_high": 0.5596757733144904,
     "right": 597,
     "n": 1125,
     "decision_score": 65.68363020338525,
     "chosen_answer_brier": 0.21699406259896153,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7855724328714279,
     "non_hallucination": 0.07262569832402235,
     "cost_per_1000": 0.022553706666666663,
     "cost_usd": 0.025372919999999997,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.03666666666666667,
     "median_latency_ms": 105.98327103070915,
     "mixed_concurrency": false
    },
    {
     "name": "Perplexity Decider 27B v1.2",
     "display_name": "Perplexity Decider 27B v1.2",
     "tag": "API · with web",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Perplexity Decider 27B v1.2 (API, with web)",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.5253333333333333,
     "wilson_low": 0.4961162598550255,
     "wilson_high": 0.5543779813210616,
     "right": 591,
     "n": 1125,
     "decision_score": 67.23592039885324,
     "chosen_answer_brier": 0.1806149253562685,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.8000754133475287,
     "non_hallucination": 0.3771043771043771,
     "cost_per_1000": 1.0295759999999998,
     "cost_usd": 1.1582729999999999,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 447.54852400001255,
     "mixed_concurrency": false
    },
    {
     "name": "SubQ Snap 1.1",
     "display_name": "SubQ Snap 1.1",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "SubQ Snap 1.1",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.5235555555555556,
     "wilson_low": 0.4943394730875544,
     "wilson_high": 0.5526113125674038,
     "right": 589,
     "n": 1125,
     "decision_score": 65.9417033512382,
     "chosen_answer_brier": 0.20472148853079167,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7872120720675063,
     "non_hallucination": 0.657185628742515,
     "cost_per_1000": 0.014915804444444443,
     "cost_usd": 0.016780279999999998,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 384.15663596242666,
     "mixed_concurrency": false
    },
    {
     "name": "Turion Jev 31B",
     "display_name": "Turion Jev 31B",
     "tag": "",
     "class_label": "Open · 31B",
     "base_model": null,
     "setup_label": "Turion Jev 31B",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.5226666666666666,
     "wilson_low": 0.49345121798248626,
     "wilson_high": 0.5517278399119073,
     "right": 588,
     "n": 1125,
     "decision_score": 66.88599632293746,
     "chosen_answer_brier": 0.18494674020791732,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7823461786949417,
     "non_hallucination": 0.015296367112810707,
     "cost_per_1000": 0.024907567793609985,
     "cost_usd": 0.028021013767811232,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.13166666666666665,
     "median_latency_ms": 385.00090898014605,
     "mixed_concurrency": false
    },
    {
     "name": "d1.2-ultra",
     "display_name": "d1.2-ultra",
     "tag": "",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "Liquid d1.2-ultra",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.5155555555555555,
     "wilson_low": 0.4863484930089186,
     "wilson_high": 0.5446567428009594,
     "right": 580,
     "n": 1125,
     "decision_score": 66.76892683249675,
     "chosen_answer_brier": 0.18017701890562055,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7998354950964884,
     "non_hallucination": 0.9450292397660819,
     "cost_per_1000": 0.012642970666666668,
     "cost_usd": 0.014223342,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 220.0660369999241,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-12B-Gen1",
     "display_name": "KnowLine-12B-Gen1",
     "tag": "",
     "class_label": "Open · 12B",
     "base_model": "Gemma 4 12B",
     "setup_label": "KnowLine-12B-Gen1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5057777777777778,
     "wilson_low": 0.4765918598680806,
     "wilson_high": 0.5349243705755883,
     "right": 569,
     "n": 1125,
     "decision_score": 65.90062336123897,
     "chosen_answer_brier": 0.18776531055299844,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7738649150977988,
     "non_hallucination": 0.08762886597938144,
     "cost_per_1000": 0.05164444444444445,
     "cost_usd": 0.058100000000000006,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.023333333333333334,
     "median_latency_ms": 68.44573598937131,
     "mixed_concurrency": false
    },
    {
     "name": "Gemini Decision 1",
     "display_name": "Gemini Decision 1",
     "tag": "20261111",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Gemini Decision 1",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.5004444444444445,
     "wilson_low": 0.4712747472059725,
     "wilson_high": 0.5296111166743097,
     "right": 563,
     "n": 1125,
     "decision_score": 65.37458458484114,
     "chosen_answer_brier": 0.19295275274762155,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7675802608041566,
     "non_hallucination": 0.01981981981981982,
     "cost_per_1000": 0.036774577777777775,
     "cost_usd": 0.0413714,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 228.39626402128488,
     "mixed_concurrency": false
    },
    {
     "name": "d1.2",
     "display_name": "d1.2",
     "tag": "",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "Liquid d1.2",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.4915555555555556,
     "wilson_low": 0.4624202429460476,
     "wilson_high": 0.5207483433285902,
     "right": 553,
     "n": 1125,
     "decision_score": 64.51203408371188,
     "chosen_answer_brier": 0.20131487388131805,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7416886910557796,
     "non_hallucination": 0.7883663366336634,
     "cost_per_1000": 0.006020462222222222,
     "cost_usd": 0.00677302,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 222.31435499998042,
     "mixed_concurrency": false
    },
    {
     "name": "Jeffan-9B-1211",
     "display_name": "Jeffan-9B-1211",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": "Qwen3.5-9B",
     "setup_label": "Jeffan-9B-1211",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.4817777777777778,
     "wilson_low": 0.4526909052709956,
     "wilson_high": 0.5109886756374331,
     "right": 542,
     "n": 1125,
     "decision_score": 64.3264601752078,
     "chosen_answer_brier": 0.19524857427362183,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7533482496059952,
     "non_hallucination": 0.23370429252782193,
     "cost_per_1000": 0.03934814814814815,
     "cost_usd": 0.04426666666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.017777777777777778,
     "median_latency_ms": 51.69780200230889,
     "mixed_concurrency": false
    },
    {
     "name": "Jev 1.14",
     "display_name": "Jev 1.14",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Jev 1.14.0",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.4817777777777778,
     "wilson_low": 0.4526909052709956,
     "wilson_high": 0.5109886756374331,
     "right": 542,
     "n": 1125,
     "decision_score": 64.13998222222222,
     "chosen_answer_brier": 0.19897813333333333,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7497404948320495,
     "non_hallucination": 0.24499229583975346,
     "cost_per_1000": 0.02564154133333333,
     "cost_usd": 0.028846734,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 165.5693930806592,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE 9B",
     "display_name": "GLiDE 9B",
     "tag": "",
     "class_label": "Open · 7.9B",
     "base_model": null,
     "setup_label": "GLiDE 9B (open)",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.4782222222222222,
     "wilson_low": 0.4491557248988175,
     "wilson_high": 0.5074369449673534,
     "right": 538,
     "n": 1125,
     "decision_score": 63.94636533023077,
     "chosen_answer_brier": 0.19929491561760673,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7492511225245879,
     "non_hallucination": 0.30381679389312977,
     "cost_per_1000": 0.013294545678867778,
     "cost_usd": 0.01495636388872625,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.07027777777777777,
     "median_latency_ms": 207.9551100032404,
     "mixed_concurrency": false
    },
    {
     "name": "celeris-1-decision",
     "display_name": "celeris-1-decision",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Celeris-1 Decision",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.4728888888888889,
     "wilson_low": 0.4438557188299771,
     "wilson_high": 0.5021065844728071,
     "right": 532,
     "n": 1125,
     "decision_score": 62.93696,
     "chosen_answer_brier": 0.21414968888888888,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7352302552333616,
     "non_hallucination": 0.9227272727272727,
     "cost_per_1000": 0.015404764444444443,
     "cost_usd": 0.01733036,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 389.8249570047483,
     "mixed_concurrency": false
    },
    {
     "name": "celeris-1.1-decision",
     "display_name": "celeris-1.1-decision",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Celeris-1.1 Decision",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.4728888888888889,
     "wilson_low": 0.4438557188299771,
     "wilson_high": 0.5021065844728071,
     "right": 532,
     "n": 1125,
     "decision_score": 62.92276,
     "chosen_answer_brier": 0.2144336888888889,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7331603671911651,
     "non_hallucination": 0.9100227790432802,
     "cost_per_1000": 0.0015404764444444444,
     "cost_usd": 0.001733036,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 382.87439299892867,
     "mixed_concurrency": false
    },
    {
     "name": "StartLux-Decision-v1.1-9B-Instant",
     "display_name": "StartLux-Decision-v1.1-9B-Instant",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": null,
     "setup_label": "StartLux-Decision v1.1 9B",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.4657777777777778,
     "wilson_low": 0.4367942128201642,
     "wilson_high": 0.49499426839810434,
     "right": 524,
     "n": 1125,
     "decision_score": 63.55219338259309,
     "chosen_answer_brier": 0.19473391012591604,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.761936213181593,
     "non_hallucination": 0.17445482866043613,
     "cost_per_1000": 0.010036593773374486,
     "cost_usd": 0.011291167995046298,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.05305555555555556,
     "median_latency_ms": 160.06175499933306,
     "mixed_concurrency": false
    },
    {
     "name": "Adjudex 9B v5",
     "display_name": "Adjudex 9B v5",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": "Qwen3.5-9B",
     "setup_label": "Adjudex 9B v5",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.464,
     "wilson_low": 0.4350297608736774,
     "wilson_high": 0.49321526482346223,
     "right": 522,
     "n": 1125,
     "decision_score": 63.47707302898673,
     "chosen_answer_brier": 0.19445853942026542,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7710839163060813,
     "non_hallucination": 0.12202852614896989,
     "cost_per_1000": 0.03135555555555555,
     "cost_usd": 0.035275,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.014166666666666666,
     "median_latency_ms": 40.6793150032172,
     "mixed_concurrency": false
    },
    {
     "name": "Elephant-1",
     "display_name": "Elephant-1",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": null,
     "setup_label": "Elephant-1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.464,
     "wilson_low": 0.4350297608736774,
     "wilson_high": 0.49321526482346223,
     "right": 522,
     "n": 1125,
     "decision_score": 63.391223575952175,
     "chosen_answer_brier": 0.19617552848095646,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7676226148948743,
     "non_hallucination": 0.7560355781448539,
     "cost_per_1000": 0.013436328888888889,
     "cost_usd": 0.01511587,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.02638888888888889,
     "median_latency_ms": 78.7134550000701,
     "mixed_concurrency": false
    },
    {
     "name": "Jev 1.13",
     "display_name": "Jev 1.13",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Jev 1.13.0",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.4631111111111111,
     "wilson_low": 0.434147673766127,
     "wilson_high": 0.49232562417044806,
     "right": 521,
     "n": 1125,
     "decision_score": 63.36038666666666,
     "chosen_answer_brier": 0.19590337777777778,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7635755233821866,
     "non_hallucination": 0.15765069551777433,
     "cost_per_1000": 0.02564154133333333,
     "cost_usd": 0.028846734,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 166.94210900459439,
     "mixed_concurrency": false
    },
    {
     "name": "d1.1",
     "display_name": "d1.1",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Liquid d1.1",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.46044444444444443,
     "wilson_low": 0.4315019683417899,
     "wilson_high": 0.48965614631309184,
     "right": 518,
     "n": 1125,
     "decision_score": 63.139299785750765,
     "chosen_answer_brier": 0.1976584487294291,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7479104781411207,
     "non_hallucination": 0.9056179775280899,
     "cost_per_1000": 0.006020462222222222,
     "cost_usd": 0.00677302,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 176.29388498608023,
     "mixed_concurrency": false
    },
    {
     "name": "StartLux-Decision-v1.1-4B-Instant",
     "display_name": "StartLux-Decision-v1.1-4B-Instant",
     "tag": "",
     "class_label": "Open · 4.0B",
     "base_model": null,
     "setup_label": "StartLux-Decision v1.1 4B",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.45866666666666667,
     "wilson_low": 0.42973862835522014,
     "wilson_high": 0.4878760307785327,
     "right": 516,
     "n": 1125,
     "decision_score": 62.63027092613061,
     "chosen_answer_brier": 0.2060612481440545,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7204847188808696,
     "non_hallucination": 0.15993788819875776,
     "cost_per_1000": 0.004624189801345314,
     "cost_usd": 0.005202213526513478,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.024444444444444446,
     "median_latency_ms": 73.16257402999327,
     "mixed_concurrency": false
    },
    {
     "name": "Dexy-Decision-9B-V2",
     "display_name": "Dexy-Decision-9B-V2",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": "vllm-sr/d3-flash",
     "setup_label": "Dexy-Decision-9B-V2",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.4533333333333333,
     "wilson_low": 0.42445083741160755,
     "wilson_high": 0.4825334551587584,
     "right": 510,
     "n": 1125,
     "decision_score": 61.812010928506474,
     "chosen_answer_brier": 0.21709311476320378,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7587342579308146,
     "non_hallucination": 0.12403100775193798,
     "cost_per_1000": 0.03381481481481482,
     "cost_usd": 0.03804166666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.015277777777777777,
     "median_latency_ms": 46.592309008701704,
     "mixed_concurrency": false
    },
    {
     "name": "Mercury Decide",
     "display_name": "Mercury Decide",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Inception Mercury Decide",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.45066666666666666,
     "wilson_low": 0.4218081979416589,
     "wilson_high": 0.4798609113470137,
     "right": 507,
     "n": 1125,
     "decision_score": 61.17502703906994,
     "chosen_answer_brier": 0.22716612588526772,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7462004429890913,
     "non_hallucination": 0.20539730134932535,
     "cost_per_1000": 0.007519022222222222,
     "cost_usd": 0.0084589,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 300.60225690249354,
     "mixed_concurrency": false
    },
    {
     "name": "Perplexity Decider Flash v1.3",
     "display_name": "Perplexity Decider Flash v1.3",
     "tag": "",
     "class_label": "Open · 4.0B",
     "base_model": null,
     "setup_label": "Perplexity Decider Flash v1.3",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.4471111111111111,
     "wilson_low": 0.41828598405130224,
     "wilson_high": 0.4762962141951126,
     "right": 503,
     "n": 1125,
     "decision_score": 62.144297687404865,
     "chosen_answer_brier": 0.20422515736301378,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7571260539655955,
     "non_hallucination": 0.2441860465116279,
     "cost_per_1000": 0.02643703703703704,
     "cost_usd": 0.02974166666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.011944444444444445,
     "median_latency_ms": 32.60860699992918,
     "mixed_concurrency": false
    },
    {
     "name": "Microsoft-Decision-1",
     "display_name": "Microsoft-Decision-1",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": "Qwen3.5-9B",
     "setup_label": "Microsoft-Decision-1",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.4444444444444444,
     "wilson_low": 0.4156453044792613,
     "wilson_high": 0.4736217104854601,
     "right": 500,
     "n": 1125,
     "decision_score": 61.5273465774752,
     "chosen_answer_brier": 0.2138975128949406,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7119504,
     "non_hallucination": 0.19765739385065886,
     "cost_per_1000": 0.013423125333333334,
     "cost_usd": 0.015101016000000002,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 315.03560091368854,
     "mixed_concurrency": false
    },
    {
     "name": "Valen-Decision-1.5-9B",
     "display_name": "Valen-Decision-1.5-9B",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": "Qwen3.5-9B",
     "setup_label": "Valen-Decision-1.5-9B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.44355555555555554,
     "wilson_low": 0.4147652650701107,
     "wilson_high": 0.47273002213404625,
     "right": 499,
     "n": 1125,
     "decision_score": 61.4211718846176,
     "chosen_answer_brier": 0.21513211786320363,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7008873977987925,
     "non_hallucination": 0.13569321533923304,
     "cost_per_1000": 0.04242222222222222,
     "cost_usd": 0.047725000000000004,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.019166666666666665,
     "median_latency_ms": 60.265301988692954,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-9B-Gen1",
     "display_name": "KnowLine-9B-Gen1",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": "Qwen3.5-9B",
     "setup_label": "KnowLine-9B-Gen1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.44266666666666665,
     "wilson_low": 0.41388531931166495,
     "wilson_high": 0.47183824013192754,
     "right": 498,
     "n": 1125,
     "decision_score": 61.97392982373859,
     "chosen_answer_brier": 0.2031880701918948,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7251397936242578,
     "non_hallucination": 0.06656580937972768,
     "cost_per_1000": 0.036274074074074084,
     "cost_usd": 0.04080833333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.01638888888888889,
     "median_latency_ms": 49.51414099923568,
     "mixed_concurrency": false
    },
    {
     "name": "vera-medium-omni",
     "display_name": "vera-medium-omni",
     "tag": "",
     "class_label": "Open · 4.2B",
     "base_model": "Qwen3.5-4B",
     "setup_label": "vera-medium-omni",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.4408888888888889,
     "wilson_low": 0.4121257089774178,
     "wilson_high": 0.4700543949450458,
     "right": 496,
     "n": 1125,
     "decision_score": 61.90377916533333,
     "chosen_answer_brier": 0.20281330558222224,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7389257141391866,
     "non_hallucination": 0.13582089552238805,
     "cost_per_1000": 0.025822222222222226,
     "cost_usd": 0.029050000000000003,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.011666666666666667,
     "median_latency_ms": 32.33273699879646,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-9B-Gen2",
     "display_name": "KnowLine-9B-Gen2",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": "Qwen3.5-9B",
     "setup_label": "KnowLine-9B-Gen2",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.4391111111111111,
     "wilson_low": 0.41036647394716497,
     "wilson_high": 0.46827017445416974,
     "right": 494,
     "n": 1125,
     "decision_score": 61.90236233710874,
     "chosen_answer_brier": 0.20106386436893622,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7300121265005742,
     "non_hallucination": 0.1951219512195122,
     "cost_per_1000": 0.039962962962962964,
     "cost_usd": 0.044958333333333336,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.018055555555555554,
     "median_latency_ms": 52.18802302260883,
     "mixed_concurrency": false
    },
    {
     "name": "Cerno 9B v1.2",
     "display_name": "Cerno 9B v1.2",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": null,
     "setup_label": "Cerno 9B v1.2",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.4328888888888889,
     "wilson_low": 0.40421211401766477,
     "wilson_high": 0.4620224400597187,
     "right": 487,
     "n": 1125,
     "decision_score": 60.40847317911111,
     "chosen_answer_brier": 0.22471942530666666,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7520051109408895,
     "non_hallucination": 0.05993690851735016,
     "cost_per_1000": 0.0133470932902467,
     "cost_usd": 0.015015479951527539,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.07055555555555555,
     "median_latency_ms": 209.3045680085197,
     "mixed_concurrency": false
    },
    {
     "name": "Perplexity Decider Flash v1.2",
     "display_name": "Perplexity Decider Flash v1.2",
     "tag": "",
     "class_label": "Open · 4.0B",
     "base_model": null,
     "setup_label": "Perplexity Decider Flash v1.2",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.4311111111111111,
     "wilson_low": 0.402454574094892,
     "wilson_high": 0.4602365244613626,
     "right": 485,
     "n": 1125,
     "decision_score": 61.34146752818315,
     "chosen_answer_brier": 0.2042817605474479,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.728659793814433,
     "non_hallucination": 0.2893258426966292,
     "cost_per_1000": 0.0037589511111111107,
     "cost_usd": 0.0042288199999999995,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.015555555555555555,
     "median_latency_ms": 45.26580194942653,
     "mixed_concurrency": false
    },
    {
     "name": "Open-Jev-27B-v1.1",
     "display_name": "Open-Jev-27B-v1.1",
     "tag": "",
     "class_label": "Open · 27B",
     "base_model": "Qwen3.8-27B + adapter",
     "setup_label": "Open-Jev 27B v1.1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.42577777777777776,
     "wilson_low": 0.397184225656079,
     "wilson_high": 0.4548765063367888,
     "right": 479,
     "n": 1125,
     "decision_score": 59.778255508487874,
     "chosen_answer_brier": 0.23021266760802017,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7334035690971257,
     "non_hallucination": 0.2818696883852691,
     "cost_per_1000": 0.2944962962962963,
     "cost_usd": 0.3313083333333333,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.13305555555555557,
     "median_latency_ms": 412.82899399993767,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine 4B Gen5.5",
     "display_name": "KnowLine 4B Gen5.5",
     "tag": "",
     "class_label": "Open · 4.0B",
     "base_model": null,
     "setup_label": "KnowLine 4B Gen5.5",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.4231111111111111,
     "wilson_low": 0.39455033271464124,
     "wilson_high": 0.45219521599653323,
     "right": 476,
     "n": 1125,
     "decision_score": 59.6640123292149,
     "chosen_answer_brier": 0.22983086452681326,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7083068974893502,
     "non_hallucination": 0.08284023668639054,
     "cost_per_1000": 0.00588533247443949,
     "cost_usd": 0.006620999033744426,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.03111111111111111,
     "median_latency_ms": 95.72846497758292,
     "mixed_concurrency": false
    },
    {
     "name": "Drex 2.0",
     "display_name": "Drex 2.0",
     "tag": "",
     "class_label": "Open · 35B MoE · 3.0B active",
     "base_model": null,
     "setup_label": "Drex 2.0",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.4186666666666667,
     "wilson_low": 0.39016241660722983,
     "wilson_high": 0.4477244933011224,
     "right": 471,
     "n": 1125,
     "decision_score": 60.733598390666664,
     "chosen_answer_brier": 0.20399469885333332,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.716586480713168,
     "non_hallucination": 0.376502002670227,
     "cost_per_1000": 0.018413706666666665,
     "cost_usd": 0.02071542,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 345.0665019918233,
     "mixed_concurrency": false
    },
    {
     "name": "Clef",
     "display_name": "Clef",
     "tag": "",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Clef",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.408,
     "wilson_low": 0.37964119338869906,
     "wilson_high": 0.4369849833928795,
     "right": 459,
     "n": 1125,
     "decision_score": 58.553709559111105,
     "chosen_answer_brier": 0.23692580881777778,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.6973787513003199,
     "non_hallucination": 0.036308623298033284,
     "cost_per_1000": 0.10631061333333333,
     "cost_usd": 0.11959944,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.0636111111111111,
     "median_latency_ms": 195.94811199976903,
     "mixed_concurrency": false
    },
    {
     "name": "d1",
     "display_name": "d1",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Liquid d1",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.408,
     "wilson_low": 0.37964119338869906,
     "wilson_high": 0.4369849833928795,
     "right": 459,
     "n": 1125,
     "decision_score": 59.759901975440854,
     "chosen_answer_brier": 0.21280196049118297,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7227178158550708,
     "non_hallucination": 0.1134453781512605,
     "cost_per_1000": 0.006020462222222222,
     "cost_usd": 0.00677302,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 188.29521094448864,
     "mixed_concurrency": false
    },
    {
     "name": "OneJev-27B",
     "display_name": "OneJev-27B",
     "tag": "",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "OneJev 27B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.4062222222222222,
     "wilson_low": 0.37788900594094155,
     "wilson_high": 0.4351937153195081,
     "right": 457,
     "n": 1125,
     "decision_score": 60.26812212415147,
     "chosen_answer_brier": 0.2008597797391929,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7259709246714449,
     "non_hallucination": 0.01188707280832095,
     "cost_per_1000": 0.08607407407407408,
     "cost_usd": 0.09683333333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.03888888888888889,
     "median_latency_ms": 107.8791750005621,
     "mixed_concurrency": false
    },
    {
     "name": "Chinese Decision Pointer 4B",
     "display_name": "Chinese Decision Pointer 4B",
     "tag": "",
     "class_label": "Open · 4.0B",
     "base_model": "Qwen3.5-4B",
     "setup_label": "Chinese Decision Pointer 4B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.40355555555555556,
     "wilson_low": 0.37526145195534033,
     "wilson_high": 0.432506086023416,
     "right": 454,
     "n": 1125,
     "decision_score": 58.90016109166391,
     "chosen_answer_brier": 0.22555233372227734,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7237143588699883,
     "non_hallucination": 0.14553314121037464,
     "cost_per_1000": 0.029511111111111113,
     "cost_usd": 0.0332,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.013333333333333334,
     "median_latency_ms": 37.97645401209593,
     "mixed_concurrency": false
    },
    {
     "name": "Verdin-4B-v1.1",
     "display_name": "Verdin-4B-v1.1",
     "tag": "",
     "class_label": "Open · 4.0B",
     "base_model": "Qwen3.5-4B",
     "setup_label": "Verdin-4B-v1.1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.3973333333333333,
     "wilson_low": 0.3691339006453455,
     "wilson_high": 0.4262315430094596,
     "right": 447,
     "n": 1125,
     "decision_score": 59.257674788596574,
     "chosen_answer_brier": 0.21217983756140177,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7289501296747243,
     "non_hallucination": 0.12642045454545456,
     "cost_per_1000": 0.0332,
     "cost_usd": 0.03735,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.015,
     "median_latency_ms": 45.41159300424624,
     "mixed_concurrency": false
    },
    {
     "name": "Winnow-12B",
     "display_name": "Winnow-12B",
     "tag": "",
     "class_label": "Open · 12B",
     "base_model": null,
     "setup_label": "Winnow-12B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.3973333333333333,
     "wilson_low": 0.3691339006453455,
     "wilson_high": 0.4262315430094596,
     "right": 447,
     "n": 1125,
     "decision_score": 58.43325769548569,
     "chosen_answer_brier": 0.22866817942361936,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7265381138101932,
     "non_hallucination": 0.29747675962815406,
     "cost_per_1000": 0.05164444444444445,
     "cost_usd": 0.058100000000000006,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.023333333333333334,
     "median_latency_ms": 65.94800899983966,
     "mixed_concurrency": false
    },
    {
     "name": "Cygnet",
     "display_name": "Cygnet",
     "tag": "",
     "class_label": "Open · 12B",
     "base_model": "Gemma 4 12B IT",
     "setup_label": "Cygnet",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.3928888888888889,
     "wilson_low": 0.3647600156359362,
     "wilson_high": 0.4217467892160467,
     "right": 442,
     "n": 1125,
     "decision_score": 58.967676356800844,
     "chosen_answer_brier": 0.21353536175287208,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.7198147645137568,
     "non_hallucination": 0.11299435028248588,
     "cost_per_1000": 0.039711911111111116,
     "cost_usd": 0.044675900000000004,
     "cost_unit": "USD",
     "cost_basis": "public host's list price (same weights)",
     "gpu_hours": 0.013888888888888888,
     "median_latency_ms": 39.896609998322674,
     "mixed_concurrency": false
    },
    {
     "name": "Clef-Flash",
     "display_name": "Clef-Flash",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": null,
     "setup_label": "Clef-flash",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.3893333333333333,
     "wilson_low": 0.3612626809585056,
     "wilson_high": 0.4181572128512194,
     "right": 438,
     "n": 1125,
     "decision_score": 57.70097677111111,
     "chosen_answer_brier": 0.23531379791111112,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.6586658291958286,
     "non_hallucination": 0.14425770308123248,
     "cost_per_1000": 0.01683251377777778,
     "cost_usd": 0.018936578000000003,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.03666666666666667,
     "median_latency_ms": 109.96608000095875,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE 2B",
     "display_name": "GLiDE 2B",
     "tag": "",
     "class_label": "Open · 1.9B",
     "base_model": null,
     "setup_label": "GLiDE 2B (open)",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.32266666666666666,
     "wilson_low": 0.29599142579280374,
     "wilson_high": 0.35054888597458694,
     "right": 363,
     "n": 1125,
     "decision_score": 54.386221345587934,
     "chosen_answer_brier": 0.23494223975490786,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.6323398624758682,
     "non_hallucination": 0.32897862232779096,
     "cost_per_1000": 0.0022595472892937333,
     "cost_usd": 0.0025419907004554498,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.011944444444444445,
     "median_latency_ms": 34.38109700800851,
     "mixed_concurrency": false
    },
    {
     "name": "Metask-Jev-4B",
     "display_name": "Metask-Jev-4B",
     "tag": "",
     "class_label": "Open · 4.5B",
     "base_model": null,
     "setup_label": "Metask-Jev 4B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.3208888888888889,
     "wilson_low": 0.29425911904249813,
     "wilson_high": 0.34873773720376366,
     "right": 361,
     "n": 1125,
     "decision_score": 56.29635965281661,
     "chosen_answer_brier": 0.19496169583255668,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.6595988455569898,
     "non_hallucination": 0.015384615384615385,
     "cost_per_1000": 0.0909925925925926,
     "cost_usd": 0.10236666666666668,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.04111111111111111,
     "median_latency_ms": 123.22193407453597,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-2B-Gen1",
     "display_name": "KnowLine-2B-Gen1",
     "tag": "",
     "class_label": "Open · 2.0B",
     "base_model": "Qwen3.5-2B",
     "setup_label": "KnowLine-2B-Gen1",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.31733333333333336,
     "wilson_low": 0.29079586118563544,
     "wilson_high": 0.34511408401836846,
     "right": 357,
     "n": 1125,
     "decision_score": 55.15217280323133,
     "chosen_answer_brier": 0.21428987726870666,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.6010409372082166,
     "non_hallucination": 0.1573170731707317,
     "cost_per_1000": 0.0037834280192825293,
     "cost_usd": 0.004256356521692845,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.02,
     "median_latency_ms": 59.10773802315816,
     "mixed_concurrency": false
    },
    {
     "name": "StartLux-Decision-v1.1-2B-Instant",
     "display_name": "StartLux-Decision-v1.1-2B-Instant",
     "tag": "",
     "class_label": "Open · 2.0B",
     "base_model": null,
     "setup_label": "StartLux-Decision v1.1 2B",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.3075555555555556,
     "wilson_low": 0.2812813415885122,
     "wilson_high": 0.3351395982492827,
     "right": 346,
     "n": 1125,
     "decision_score": 55.50501290495344,
     "chosen_answer_brier": 0.1974552974564868,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.639948948926666,
     "non_hallucination": 0.3136792452830189,
     "cost_per_1000": 0.002522285346188353,
     "cost_usd": 0.002837571014461897,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.013333333333333334,
     "median_latency_ms": 39.87727497587912,
     "mixed_concurrency": false
    },
    {
     "name": "d1-2B",
     "display_name": "d1-2B",
     "tag": "",
     "class_label": "Open · 1.9B",
     "base_model": "LFM3-VL-2B",
     "setup_label": "Liquid d1-2B (open weights)",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.30666666666666664,
     "wilson_low": 0.2804170802710874,
     "wilson_high": 0.3342321318061429,
     "right": 345,
     "n": 1125,
     "decision_score": 55.04533868651795,
     "chosen_answer_brier": 0.20575989293630767,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.6199126718691936,
     "non_hallucination": 0.8645731108930323,
     "cost_per_1000": 0.014755555555555556,
     "cost_usd": 0.0166,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.006666666666666667,
     "median_latency_ms": 17.1983860200271,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE 0.8B",
     "display_name": "GLiDE 0.8B",
     "tag": "",
     "class_label": "Open · 0.8B",
     "base_model": null,
     "setup_label": "GLiDE 0.8B (open)",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.29155555555555557,
     "wilson_low": 0.265742782608576,
     "wilson_high": 0.31878705753905867,
     "right": 328,
     "n": 1125,
     "decision_score": 53.53005592225997,
     "chosen_answer_brier": 0.2209544371103561,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.6170165559873917,
     "non_hallucination": 0.2607184241019699,
     "cost_per_1000": 0.0015764283413677205,
     "cost_usd": 0.0017734818840386857,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.008333333333333333,
     "median_latency_ms": 23.113314004149288,
     "mixed_concurrency": false
    },
    {
     "name": "OneJev1.6-0.8B",
     "display_name": "OneJev1.6-0.8B",
     "tag": "",
     "class_label": "Open · 0.8B",
     "base_model": null,
     "setup_label": "OneJev1.6-0.8B",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.2702222222222222,
     "wilson_low": 0.24508668277160642,
     "wilson_high": 0.2969216911224811,
     "right": 304,
     "n": 1125,
     "decision_score": 52.71724789589124,
     "chosen_answer_brier": 0.21587726430439733,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.5445461247515866,
     "non_hallucination": 0.04600484261501211,
     "cost_per_1000": 0.0011560474503363284,
     "cost_usd": 0.0013005533816283694,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.006111111111111111,
     "median_latency_ms": 15.75754300574772,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-0.8B-Gen1",
     "display_name": "KnowLine-0.8B-Gen1",
     "tag": "",
     "class_label": "Open · 0.8B",
     "base_model": "Qwen3.5-0.8B",
     "setup_label": "KnowLine-0.8B-Gen1",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.26666666666666666,
     "wilson_low": 0.24165118992849527,
     "wilson_high": 0.29327027292333446,
     "right": 300,
     "n": 1125,
     "decision_score": 53.45296593455431,
     "chosen_answer_brier": 0.1976073479755805,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.5727313131313131,
     "non_hallucination": 0.011976047904191617,
     "cost_per_1000": 0.001996809232399113,
     "cost_usd": 0.002246410386449002,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.010555555555555556,
     "median_latency_ms": 31.39483797713183,
     "mixed_concurrency": false
    },
    {
     "name": "StartLux-Decision-v1.1-0.8B-Instant",
     "display_name": "StartLux-Decision-v1.1-0.8B-Instant",
     "tag": "",
     "class_label": "Open · 0.8B",
     "base_model": "Qwen3.5-0.8B",
     "setup_label": "StartLux-Decision-v1.1-0.8B-Instant",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.24711111111111111,
     "wilson_low": 0.22279466565541087,
     "wilson_high": 0.27314878646400076,
     "right": 278,
     "n": 1125,
     "decision_score": 53.2205286644792,
     "chosen_answer_brier": 0.1827005378215272,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.6412220872652528,
     "non_hallucination": 0.21380090497737556,
     "cost_per_1000": 0.0017866187868834167,
     "cost_usd": 0.0020099461352438438,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.009444444444444445,
     "median_latency_ms": 27.379673992982134,
     "mixed_concurrency": false
    },
    {
     "name": "GLiClass v4 Edge",
     "display_name": "GLiClass v4 Edge",
     "tag": "",
     "class_label": "Open · 0.4B",
     "base_model": "DeBERTa-v3-large",
     "setup_label": "GLiClass v4 Edge",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.23555555555555555,
     "wilson_low": 0.21168435487911794,
     "wilson_high": 0.2612266363529557,
     "right": 265,
     "n": 1125,
     "decision_score": 51.98585229780819,
     "chosen_answer_brier": 0.19583850959939184,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.5269986836331725,
     "non_hallucination": 0.12372304199772985,
     "cost_per_1000": 0.0013136902844731004,
     "cost_usd": 0.001477901570032238,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.006944444444444444,
     "median_latency_ms": 15.218534972518682,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE 0.4B",
     "display_name": "GLiDE 0.4B",
     "tag": "",
     "class_label": "Open · 0.4B",
     "base_model": null,
     "setup_label": "GLiDE 0.4B (open)",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.232,
     "wilson_low": 0.20827083999583704,
     "wilson_high": 0.25755324019397885,
     "right": 261,
     "n": 1125,
     "decision_score": 52.18747785579388,
     "chosen_answer_brier": 0.18825044288412246,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.5556797218674613,
     "non_hallucination": 0.7606589147286822,
     "cost_per_1000": 0.0009458570048206323,
     "cost_usd": 0.0010640891304232113,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.005,
     "median_latency_ms": 13.159042980987579,
     "mixed_concurrency": false
    },
    {
     "name": "Strands Decider 2B",
     "display_name": "Strands Decider 2B",
     "tag": "",
     "class_label": "Open · 2.0B",
     "base_model": null,
     "setup_label": "Strands Decider 2B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.224,
     "wilson_low": 0.20059940687135164,
     "wilson_high": 0.2492791234733841,
     "right": 252,
     "n": 1125,
     "decision_score": 51.86198021911111,
     "chosen_answer_brier": 0.18676039561777777,
     "n_scored": 1125,
     "decision_score_lacking": 0,
     "auroc": 0.567980781468754,
     "non_hallucination": 0.08286038592508513,
     "cost_per_1000": 0.02766666666666667,
     "cost_usd": 0.031125000000000003,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.0125,
     "median_latency_ms": 36.09124897047877,
     "mixed_concurrency": false
    }
   ]
  },
  "math": {
   "name": "Math",
   "questions": 2163,
   "setups": [
    {
     "name": "Claude Opus 5.5",
     "display_name": "Claude Opus 5.5",
     "tag": "chat",
     "class_label": "Closed · Flagship",
     "base_model": null,
     "setup_label": "Claude Opus 5.5",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9976883957466481,
     "wilson_low": 0.9945998412576655,
     "wilson_high": 0.9990122437235347,
     "right": 2158,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.625,
     "cost_per_1000": 9.870942209893665,
     "cost_usd": 21.350848,
     "cost_unit": "USD",
     "cost_basis": "Claude subscription",
     "gpu_hours": null,
     "median_latency_ms": 3200.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen4-Max",
     "display_name": "Qwen4-Max",
     "tag": "high · chat",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "Qwen4-Max",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9976883957466481,
     "wilson_low": 0.9945998412576655,
     "wilson_high": 0.9990122437235347,
     "right": 2158,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.5454545454545454,
     "cost_per_1000": 1.478152288488211,
     "cost_usd": 3.1972434,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 4710.0,
     "mixed_concurrency": false
    },
    {
     "name": "Claude Fable 5.1",
     "display_name": "Claude Fable 5.1",
     "tag": "chat",
     "class_label": "Closed · Flagship",
     "base_model": null,
     "setup_label": "Claude Fable 5.1",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9972260748959778,
     "wilson_low": 0.993960980972945,
     "wilson_high": 0.9987281016069722,
     "right": 2157,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.6,
     "cost_per_1000": 0.0,
     "cost_usd": 0.0,
     "cost_unit": "USD",
     "cost_basis": "Claude subscription",
     "gpu_hours": null,
     "median_latency_ms": 5030.0,
     "mixed_concurrency": false
    },
    {
     "name": "Claude Sonnet 5.5",
     "display_name": "Claude Sonnet 5.5",
     "tag": "chat",
     "class_label": "Closed · Workhorse",
     "base_model": null,
     "setup_label": "Claude Sonnet 5.5",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9967637540453075,
     "wilson_low": 0.993334581898462,
     "wilson_high": 0.9984314982801723,
     "right": 2156,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.625,
     "cost_per_1000": 4.734299583911234,
     "cost_usd": 10.24029,
     "cost_unit": "USD",
     "cost_basis": "Claude subscription",
     "gpu_hours": null,
     "median_latency_ms": 3460.0,
     "mixed_concurrency": false
    },
    {
     "name": "DeepSeek-V4.1-Flash",
     "display_name": "DeepSeek-V4.1-Flash",
     "tag": "high · chat · 1017",
     "class_label": "Open · 552B MoE · 16B active",
     "base_model": null,
     "setup_label": "DeepSeek-V4.1-Flash 1017",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.996301433194637,
     "wilson_low": 0.9927183626130163,
     "wilson_high": 0.9981247151643349,
     "right": 2155,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.2,
     "cost_per_1000": 0.2274187406380028,
     "cost_usd": 0.491906736,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 1910.0,
     "mixed_concurrency": false
    },
    {
     "name": "GPT-6 Astra",
     "display_name": "GPT-6 Astra",
     "tag": "chat",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "GPT-6 Astra",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.996301433194637,
     "wilson_low": 0.9927183626130163,
     "wilson_high": 0.9981247151643349,
     "right": 2155,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.7857142857142857,
     "cost_per_1000": 0.0,
     "cost_usd": 0.0,
     "cost_unit": "USD",
     "cost_basis": "ChatGPT subscription",
     "gpu_hours": null,
     "median_latency_ms": 5280.0,
     "mixed_concurrency": false
    },
    {
     "name": "GPT-6.1 Sol",
     "display_name": "GPT-6.1 Sol",
     "tag": "chat",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "GPT-6.1 Sol",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.996301433194637,
     "wilson_low": 0.9927183626130163,
     "wilson_high": 0.9981247151643349,
     "right": 2155,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.7857142857142857,
     "cost_per_1000": 0.0,
     "cost_usd": 0.0,
     "cost_unit": "USD",
     "cost_basis": "ChatGPT subscription",
     "gpu_hours": null,
     "median_latency_ms": 4810.0,
     "mixed_concurrency": false
    },
    {
     "name": "Inkling-Small",
     "display_name": "Inkling-Small",
     "tag": "high · chat · Jul 2026",
     "class_label": "Open · 276B MoE · 12B active",
     "base_model": null,
     "setup_label": "Inkling Small",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.996301433194637,
     "wilson_low": 0.9927183626130163,
     "wilson_high": 0.9981247151643349,
     "right": 2155,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.5454545454545454,
     "cost_per_1000": 0.9538116042533519,
     "cost_usd": 2.0630945,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 2029.9999999999998,
     "mixed_concurrency": false
    },
    {
     "name": "Gemini 4 Carbon",
     "display_name": "Gemini 4 Carbon",
     "tag": "chat",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "Gemini 4 Carbon (Antigravity subscription)",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9958391123439667,
     "wilson_low": 0.9921106643770949,
     "wilson_high": 0.9978094109989732,
     "right": 2154,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.75,
     "cost_per_1000": null,
     "cost_usd": null,
     "cost_unit": "USD",
     "cost_basis": null,
     "gpu_hours": null,
     "median_latency_ms": 13410.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemini 4 Hydrogen",
     "display_name": "Gemini 4 Hydrogen",
     "tag": "chat",
     "class_label": "",
     "base_model": null,
     "setup_label": "Gemini 4 Hydrogen (Antigravity subscription)",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9958391123439667,
     "wilson_low": 0.9921106643770949,
     "wilson_high": 0.9978094109989732,
     "right": 2154,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.8571428571428571,
     "cost_per_1000": null,
     "cost_usd": null,
     "cost_unit": "USD",
     "cost_basis": null,
     "gpu_hours": null,
     "median_latency_ms": 4400.0,
     "mixed_concurrency": false
    },
    {
     "name": "Muse Ember",
     "display_name": "Muse Ember",
     "tag": "max · chat",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Muse Ember",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9958391123439667,
     "wilson_low": 0.9921106643770949,
     "wilson_high": 0.9978094109989732,
     "right": 2154,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.6428571428571429,
     "cost_per_1000": 0.3645816495607952,
     "cost_usd": 0.788590108,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 1260.0,
     "mixed_concurrency": false
    },
    {
     "name": "Grok 4.7",
     "display_name": "Grok 4.7",
     "tag": "chat",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "Grok 4.7",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9953767914932964,
     "wilson_low": 0.9915102362993337,
     "wilson_high": 0.9974868366754516,
     "right": 2153,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.975,
     "cost_per_1000": 12.061260286638927,
     "cost_usd": 26.088506,
     "cost_unit": "USD",
     "cost_basis": "xAI subscription",
     "gpu_hours": null,
     "median_latency_ms": 11100.0,
     "mixed_concurrency": false
    },
    {
     "name": "Inkling 2 Small",
     "display_name": "Inkling 2 Small",
     "tag": "high · chat",
     "class_label": "Open · 594B MoE · 17B active",
     "base_model": null,
     "setup_label": "Inkling 2 Small",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9953767914932964,
     "wilson_low": 0.9915102362993337,
     "wilson_high": 0.9974868366754516,
     "right": 2153,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.6153846153846154,
     "cost_per_1000": 0.6384771428571429,
     "cost_usd": 1.38102606,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 2110.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen3.8 Flash",
     "display_name": "Qwen3.8 Flash",
     "tag": "off · chat",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "Qwen3.8 Flash",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.994914470642626,
     "wilson_low": 0.990916107343996,
     "wilson_high": 0.997157963229506,
     "right": 2152,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.16666666666666666,
     "cost_per_1000": 0.39692356911696713,
     "cost_usd": 0.85854568,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 5740.0,
     "mixed_concurrency": false
    },
    {
     "name": "MiMo-V2.6-Flash",
     "display_name": "MiMo-V2.6-Flash",
     "tag": "on · chat",
     "class_label": "Open · 309B MoE · 15B active",
     "base_model": null,
     "setup_label": "MiMo V2.6 Flash",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9944521497919556,
     "wilson_low": 0.9903275057772982,
     "wilson_high": 0.9968235623949209,
     "right": 2151,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.5,
     "cost_per_1000": 0.21184553268608416,
     "cost_usd": 0.4582218872,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 8200.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen3.5-4B",
     "display_name": "Qwen3.5-4B",
     "tag": "on · chat",
     "class_label": "Open · 4.7B",
     "base_model": null,
     "setup_label": "Qwen3.5-4B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9944521497919556,
     "wilson_low": 0.9903275057772982,
     "wilson_high": 0.9968235623949209,
     "right": 2151,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.21428571428571427,
     "cost_per_1000": 1.2710933888118354,
     "cost_usd": 2.749375,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 1.1041666666666667,
     "median_latency_ms": 46630.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 31B",
     "display_name": "Gemma 4 31B",
     "tag": "chat",
     "class_label": "Open · 31B",
     "base_model": null,
     "setup_label": "Gemma 4 31B (chat, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9939898289412853,
     "wilson_low": 0.9897438061828758,
     "wilson_high": 0.9964842595880603,
     "right": 2150,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.21428571428571427,
     "cost_per_1000": 0.5308213900446911,
     "cost_usd": 1.1481666666666668,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.46111111111111114,
     "median_latency_ms": 11110.0,
     "mixed_concurrency": false
    },
    {
     "name": "DeepSeek-V4-Flash",
     "display_name": "DeepSeek-V4-Flash",
     "tag": "chat",
     "class_label": "Open · 285B MoE · 13B active",
     "base_model": null,
     "setup_label": "deepseek-v4-flash-dspark",
     "group": "chat",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.9930651872399445,
     "wilson_low": 0.9885891367219977,
     "wilson_high": 0.9957929242463721,
     "right": 2148,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": null,
     "cost_per_1000": 0.2948150575300555,
     "cost_usd": 0.6376849694375101,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 2.9963888888888888,
     "median_latency_ms": 8130.000000000001,
     "mixed_concurrency": false
    },
    {
     "name": "MiniCPM5-2B",
     "display_name": "MiniCPM5-2B",
     "tag": "on · chat",
     "class_label": "Open · 2.5B",
     "base_model": null,
     "setup_label": "MiniCPM5-2B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9922946524888273,
     "wilson_low": null,
     "wilson_high": null,
     "right": null,
     "n": null,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.5357142857142857,
     "cost_per_1000": 0.40856192530949814,
     "cost_usd": 0.8977833333333335,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.3605555555555556,
     "median_latency_ms": 12536.666666666666,
     "mixed_concurrency": false,
     "seed_runs": [
      {
       "seed": 0,
       "run_id": "minicpm5-2b_en",
       "right": 2147,
       "n": 2163
      },
      {
       "seed": 1,
       "run_id": "minicpm5-2b-seed1_en",
       "right": 2145,
       "n": 2163
      },
      {
       "seed": 2,
       "run_id": "minicpm5-2b-seed2_en",
       "right": 2147,
       "n": 2163
      }
     ],
     "seeds_complete": "3/3",
     "accuracy_basis": "mean of 3 seeds",
     "seed_min": 0.9916782246879334,
     "seed_max": 0.9926028663892742
    },
    {
     "name": "GPT-6.1 Terra",
     "display_name": "GPT-6.1 Terra",
     "tag": "chat",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "GPT-6.1 Terra",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9921405455386038,
     "wilson_low": 0.9874488890221289,
     "wilson_high": 0.9950871671436747,
     "right": 2146,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.7777777777777778,
     "cost_per_1000": 0.0,
     "cost_usd": 0.0,
     "cost_unit": "USD",
     "cost_basis": "ChatGPT subscription",
     "gpu_hours": null,
     "median_latency_ms": 4080.0,
     "mixed_concurrency": false
    },
    {
     "name": "Granite-4.2-3B",
     "display_name": "Granite-4.2-3B",
     "tag": "high · chat",
     "class_label": "Open · 3.0B",
     "base_model": "Granite-4.1-3B-Base",
     "setup_label": "Granite-4.2-3B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9921405455386038,
     "wilson_low": 0.9874488890221289,
     "wilson_high": 0.9950871671436747,
     "right": 2146,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.6153846153846154,
     "cost_per_1000": 0.7808830328247804,
     "cost_usd": 1.6890500000000002,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.6783333333333333,
     "median_latency_ms": 15340.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen3.8-27B",
     "display_name": "Qwen3.8-27B",
     "tag": "low · chat",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Qwen3.8-27B (chat, reasoning low, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9921405455386038,
     "wilson_low": 0.9874488890221289,
     "wilson_high": 0.9950871671436747,
     "right": 2146,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.13043478260869565,
     "cost_per_1000": 0.52986207427955,
     "cost_usd": 1.1460916666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.4602777777777778,
     "median_latency_ms": 10020.0,
     "mixed_concurrency": false
    },
    {
     "name": "LFM2.5-24B-A2B",
     "display_name": "LFM2.5-24B-A2B",
     "tag": "on · chat",
     "class_label": "Open · 24B MoE · 2.5B active",
     "base_model": null,
     "setup_label": "LFM2.5-24B-A2B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9902912621359223,
     "wilson_low": 0.985202886676591,
     "wilson_high": 0.9936411598840806,
     "right": 2142,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.7142857142857143,
     "cost_per_1000": 0.9391701340730468,
     "cost_usd": 2.0314250000000005,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.8158333333333333,
     "median_latency_ms": 27950.0,
     "mixed_concurrency": false
    },
    {
     "name": "LFM3-1.7B",
     "display_name": "LFM3-1.7B",
     "tag": "off · chat · NVIDIA RTX PRO 6000 Blackwell",
     "class_label": "Open · 1.7B",
     "base_model": null,
     "setup_label": "LFM3-1.7B (chat, thinking off, RTX PRO 6000, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9893666204345816,
     "wilson_low": null,
     "wilson_high": null,
     "right": null,
     "n": null,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": null,
     "cost_per_1000": 0.14176555195972673,
     "cost_usd": 0.3181666666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.12777777777777777,
     "median_latency_ms": 3990.0,
     "mixed_concurrency": false,
     "seed_runs": [
      {
       "seed": 0,
       "run_id": "lfm3-1.7b-vllm-nothink-rtx-seed0_en",
       "right": 2142,
       "n": 2163
      },
      {
       "seed": 1,
       "run_id": "lfm3-1.7b-vllm-nothink-rtx-seed1_en",
       "right": 2136,
       "n": 2163
      },
      {
       "seed": 2,
       "run_id": "lfm3-1.7b-vllm-nothink-rtx-seed2_en",
       "right": 2142,
       "n": 2163
      }
     ],
     "seeds_complete": "3/3",
     "accuracy_basis": "mean of 3 seeds",
     "seed_min": 0.9875173370319001,
     "seed_max": 0.9902912621359223
    },
    {
     "name": "LFM3-1.7B",
     "display_name": "LFM3-1.7B",
     "tag": "on · chat",
     "class_label": "Open · 1.7B",
     "base_model": null,
     "setup_label": "LFM3-1.7B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9893666204345816,
     "wilson_low": null,
     "wilson_high": null,
     "right": null,
     "n": null,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.6764705882352942,
     "cost_per_1000": 0.1323855755894591,
     "cost_usd": 0.28081666666666666,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.11277777777777778,
     "median_latency_ms": 3893.3333333333335,
     "mixed_concurrency": false,
     "seed_runs": [
      {
       "seed": 0,
       "run_id": "lfm3-1.7b-vllm_en",
       "right": 2139,
       "n": 2163
      },
      {
       "seed": 1,
       "run_id": "lfm3-1.7b-vllm-seed1_en",
       "right": 2143,
       "n": 2163
      },
      {
       "seed": 2,
       "run_id": "lfm3-1.7b-vllm-seed2_en",
       "right": 2138,
       "n": 2163
      }
     ],
     "seeds_complete": "3/3",
     "accuracy_basis": "mean of 3 seeds",
     "seed_min": 0.9884419787332409,
     "seed_max": 0.9907535829865927
    },
    {
     "name": "Gemma 4 12B Unified",
     "display_name": "Gemma 4 12B Unified",
     "tag": "chat",
     "class_label": "Open · 12B",
     "base_model": null,
     "setup_label": "Gemma 4 12B (chat, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9884419787332409,
     "wilson_low": 0.9829929736848968,
     "wilson_high": 0.9921590632706427,
     "right": 2138,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.14285714285714285,
     "cost_per_1000": 0.4671867776236709,
     "cost_usd": 1.0105250000000001,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.4058333333333333,
     "median_latency_ms": 19100.0,
     "mixed_concurrency": false
    },
    {
     "name": "GLM-5.3-Flash",
     "display_name": "GLM-5.3-Flash",
     "tag": "high · chat",
     "class_label": "Open · 320B MoE · 18B active",
     "base_model": null,
     "setup_label": "GLM-5.3-Flash",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9879796578825705,
     "wilson_low": 0.9824451270921871,
     "wilson_high": 0.9917839074620692,
     "right": 2137,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.17391304347826086,
     "cost_per_1000": 0.16649577438742488,
     "cost_usd": 0.36013036,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 3800.0,
     "mixed_concurrency": false
    },
    {
     "name": "LFM3-1.7B",
     "display_name": "LFM3-1.7B",
     "tag": "off · chat",
     "class_label": "Open · 1.7B",
     "base_model": null,
     "setup_label": "LFM3-1.7B (chat, thinking off, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.9870550161812298,
     "wilson_low": 0.9813542899137543,
     "wilson_high": 0.9910287398379362,
     "right": 2135,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.6774193548387096,
     "cost_per_1000": 0.05993598045456678,
     "cost_usd": 0.12964152572322793,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.6091666666666666,
     "median_latency_ms": 19570.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 E4B",
     "display_name": "Gemma 4 E4B",
     "tag": "on · chat · 32k cap · sampled",
     "class_label": "Open · 8.0B",
     "base_model": null,
     "setup_label": "Gemma 4 E4B (chat, thinking on, 32k, sampled, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9853598397287717,
     "wilson_low": null,
     "wilson_high": null,
     "right": null,
     "n": null,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.45454545454545453,
     "cost_per_1000": 0.31764010890224487,
     "cost_usd": 0.7034250000000001,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.2825,
     "median_latency_ms": 11190.0,
     "mixed_concurrency": false,
     "seed_runs": [
      {
       "seed": 0,
       "run_id": "gemma-4-e4b-it-think-32k-sampled-seed0_en",
       "right": 2136,
       "n": 2163
      },
      {
       "seed": 1,
       "run_id": "gemma-4-e4b-it-think-32k-sampled-seed1_en",
       "right": 2130,
       "n": 2163
      },
      {
       "seed": 2,
       "run_id": "gemma-4-e4b-it-think-32k-sampled-seed2_en",
       "right": 2128,
       "n": 2163
      }
     ],
     "seeds_complete": "3/3",
     "accuracy_basis": "mean of 3 seeds",
     "seed_min": 0.9838187702265372,
     "seed_max": 0.9875173370319001
    },
    {
     "name": "Gemma 4 E4B",
     "display_name": "Gemma 4 E4B",
     "tag": "on · chat · 32k cap",
     "class_label": "Open · 8.0B",
     "base_model": null,
     "setup_label": "Gemma 4 E4B (chat, thinking on, 32k, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9852057327785483,
     "wilson_low": 0.9791898580159887,
     "wilson_high": 0.9895011621305693,
     "right": 2131,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": null,
     "cost_per_1000": 0.5180305131761442,
     "cost_usd": 1.1205,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.45,
     "median_latency_ms": 20640.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen3.5-4B",
     "display_name": "Qwen3.5-4B",
     "tag": "off · chat",
     "class_label": "Open · 4.7B",
     "base_model": null,
     "setup_label": "Qwen3.5-4B (chat, thinking off, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.984743411927878,
     "wilson_low": 0.9786519578896127,
     "wilson_high": 0.9891160598556623,
     "right": 2130,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.36,
     "cost_per_1000": 0.29994606256742185,
     "cost_usd": 0.6487833333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.26055555555555554,
     "median_latency_ms": 10960.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 E4B",
     "display_name": "Gemma 4 E4B",
     "tag": "on · chat",
     "class_label": "Open · 8.0B",
     "base_model": null,
     "setup_label": "Gemma 4 E4B (chat, thinking on, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.9824318076745261,
     "wilson_low": 0.9759790877364918,
     "wilson_high": 0.987173918002368,
     "right": 2125,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.3695652173913043,
     "cost_per_1000": 0.23285661353073822,
     "cost_usd": 0.5036688550669868,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 2.3666666666666667,
     "median_latency_ms": 108720.0,
     "mixed_concurrency": false
    },
    {
     "name": "MiniMax-M3.1-Flash",
     "display_name": "MiniMax-M3.1-Flash",
     "tag": "high · chat",
     "class_label": "Open · 428B MoE · 23B active",
     "base_model": null,
     "setup_label": "MiniMax-M3.1-Flash",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9824318076745261,
     "wilson_low": 0.9759790877364918,
     "wilson_high": 0.987173918002368,
     "right": 2125,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.40625,
     "cost_per_1000": 0.10285200069348127,
     "cost_usd": 0.2224688775,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 1670.0,
     "mixed_concurrency": false
    },
    {
     "name": "MiniCPM5-2B",
     "display_name": "MiniCPM5-2B",
     "tag": "off · chat",
     "class_label": "Open · 2.5B",
     "base_model": null,
     "setup_label": "MiniCPM5-2B (chat, thinking off, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9815071659731854,
     "wilson_low": null,
     "wilson_high": null,
     "right": null,
     "n": null,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.3103448275862069,
     "cost_per_1000": 0.21883058509272102,
     "cost_usd": 0.4800166666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.19277777777777777,
     "median_latency_ms": 6353.333333333333,
     "mixed_concurrency": false,
     "seed_runs": [
      {
       "seed": 0,
       "run_id": "minicpm5-2b-nothink_en",
       "right": 2126,
       "n": 2163
      },
      {
       "seed": 1,
       "run_id": "minicpm5-2b-nothink-seed1_en",
       "right": 2122,
       "n": 2163
      },
      {
       "seed": 2,
       "run_id": "minicpm5-2b-nothink-seed2_en",
       "right": 2121,
       "n": 2163
      }
     ],
     "seeds_complete": "3/3",
     "accuracy_basis": "mean of 3 seeds",
     "seed_min": 0.9805825242718447,
     "seed_max": 0.9828941285251965
    },
    {
     "name": "Gemma 4 E4B",
     "display_name": "Gemma 4 E4B",
     "tag": "chat",
     "class_label": "Open · 8.0B",
     "base_model": null,
     "setup_label": "Gemma 4 E4B (chat, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.981044845122515,
     "wilson_low": 0.9743871739547167,
     "wilson_high": 0.9859968245802939,
     "right": 2122,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.44,
     "cost_per_1000": 0.27116658961319157,
     "cost_usd": 0.5865333333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.23555555555555555,
     "median_latency_ms": 10770.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 E2B",
     "display_name": "Gemma 4 E2B",
     "tag": "on · chat · 32k cap",
     "class_label": "Open · 5.1B",
     "base_model": null,
     "setup_label": "Gemma 4 E2B (chat, thinking on, 32k, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9782709200184928,
     "wilson_low": 0.9712257788315998,
     "wilson_high": 0.9836202052957125,
     "right": 2116,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": null,
     "cost_per_1000": 0.4096278317152104,
     "cost_usd": 0.8860250000000001,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.35583333333333333,
     "median_latency_ms": 14870.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 E2B",
     "display_name": "Gemma 4 E2B",
     "tag": "on · chat · 32k cap · sampled",
     "class_label": "Open · 5.1B",
     "base_model": null,
     "setup_label": "Gemma 4 E2B (chat, thinking on, 32k, sampled, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9767298505162584,
     "wilson_low": null,
     "wilson_high": null,
     "right": null,
     "n": null,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.22950819672131148,
     "cost_per_1000": 0.19580700672933685,
     "cost_usd": 0.4219166666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.16944444444444445,
     "median_latency_ms": 7136.666666666667,
     "mixed_concurrency": false,
     "seed_runs": [
      {
       "seed": 0,
       "run_id": "gemma-4-e2b-it-think-32k-sampled-seed0_en",
       "right": 2111,
       "n": 2163
      },
      {
       "seed": 1,
       "run_id": "gemma-4-e2b-it-think-32k-sampled-seed1_en",
       "right": 2116,
       "n": 2163
      },
      {
       "seed": 2,
       "run_id": "gemma-4-e2b-it-think-32k-sampled-seed2_en",
       "right": 2111,
       "n": 2163
      }
     ],
     "seeds_complete": "3/3",
     "accuracy_basis": "mean of 3 seeds",
     "seed_min": 0.9759593157651411,
     "seed_max": 0.9782709200184928
    },
    {
     "name": "Raon-1 nano",
     "display_name": "Raon-1 nano",
     "tag": "on · chat",
     "class_label": "Open · 3.1B",
     "base_model": "Raon-1 mini preview (MoE-to-Dense pruning and distillation)",
     "setup_label": "Raon-1 nano (chat, thinking on, KRAFTON vLLM fork, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9699491447064262,
     "wilson_low": 0.9618792305627498,
     "wilson_high": 0.9763527103414676,
     "right": 2098,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": null,
     "cost_per_1000": 1.670168747110495,
     "cost_usd": 3.612575,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 1.4508333333333334,
     "median_latency_ms": 34320.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 E2B",
     "display_name": "Gemma 4 E2B",
     "tag": "on · chat",
     "class_label": "Open · 5.1B",
     "base_model": null,
     "setup_label": "Gemma 4 E2B (chat, thinking on, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.9694868238557559,
     "wilson_low": 0.9613648157758767,
     "wilson_high": 0.9759441227270578,
     "right": 2097,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.16455696202531644,
     "cost_per_1000": 0.13804680222344584,
     "cost_usd": 0.29859523320931336,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 1.4030555555555555,
     "median_latency_ms": 64440.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 E2B",
     "display_name": "Gemma 4 E2B",
     "tag": "chat",
     "class_label": "Open · 5.1B",
     "base_model": null,
     "setup_label": "Gemma 4 E2B (chat, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9676375404530745,
     "wilson_low": 0.959311394576175,
     "wilson_high": 0.9743055343216275,
     "right": 2093,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.25333333333333335,
     "cost_per_1000": 0.20817152103559874,
     "cost_usd": 0.4502750000000001,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.18083333333333335,
     "median_latency_ms": 8119.999999999999,
     "mixed_concurrency": false
    },
    {
     "name": "LFM2.5-8B-A1B",
     "display_name": "LFM2.5-8B-A1B",
     "tag": "on · chat",
     "class_label": "Open · 8.3B MoE · 1.5B active",
     "base_model": null,
     "setup_label": "LFM2.5-8B-A1B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9625520110957004,
     "wilson_low": 0.9536959126257067,
     "wilson_high": 0.9697679898579821,
     "right": 2082,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.45161290322580644,
     "cost_per_1000": 0.48221605794421335,
     "cost_usd": 1.0430333333333335,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.41888888888888887,
     "median_latency_ms": 14530.0,
     "mixed_concurrency": true
    },
    {
     "name": "Qwen3-14B",
     "display_name": "Qwen3-14B",
     "tag": "off · chat",
     "class_label": "Open · 15B",
     "base_model": null,
     "setup_label": "Qwen3-14B (chat, reasoning off, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9579288025889967,
     "wilson_low": 0.9486252219864938,
     "wilson_high": 0.9656086564843643,
     "right": 2072,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.39316239316239315,
     "cost_per_1000": 0.3549468331021729,
     "cost_usd": 0.76775,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.30833333333333335,
     "median_latency_ms": 8039.999999999999,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen3.5-2B",
     "display_name": "Qwen3.5-2B",
     "tag": "on · chat",
     "class_label": "Open · 2.0B",
     "base_model": null,
     "setup_label": "Qwen3.5-2B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9472954230235784,
     "wilson_low": 0.9370607883885885,
     "wilson_high": 0.9559440348527598,
     "right": 2049,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.203125,
     "cost_per_1000": 2.469918323316382,
     "cost_usd": 5.342433333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 2.1455555555555557,
     "median_latency_ms": 98390.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen3.5-2B",
     "display_name": "Qwen3.5-2B",
     "tag": "off · chat",
     "class_label": "Open · 2.0B",
     "base_model": null,
     "setup_label": "Qwen3.5-2B (chat, thinking off, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9463707813222376,
     "wilson_low": 0.9360606572509331,
     "wilson_high": 0.9550981611878488,
     "right": 2047,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.31297709923664124,
     "cost_per_1000": 0.3632609030667283,
     "cost_usd": 0.7857333333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.31555555555555553,
     "median_latency_ms": 11200.0,
     "mixed_concurrency": false
    },
    {
     "name": "LFM2.5-2.6B",
     "display_name": "LFM2.5-2.6B",
     "tag": "on · chat",
     "class_label": "Open · 2.7B",
     "base_model": null,
     "setup_label": "LFM2.5-2.6B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9398982894128525,
     "wilson_low": 0.9290805323228347,
     "wilson_high": 0.9491562524979845,
     "right": 2033,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.26373626373626374,
     "cost_per_1000": 0.29994606256742185,
     "cost_usd": 0.6487833333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.26055555555555554,
     "median_latency_ms": 9860.0,
     "mixed_concurrency": false
    },
    {
     "name": "GPT-6 Luna",
     "display_name": "GPT-6 Luna",
     "tag": "chat",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "GPT-6 Luna (low)",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.93666204345816,
     "wilson_low": 0.9256029908331322,
     "wilson_high": 0.9461727771787058,
     "right": 2026,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.6891891891891891,
     "cost_per_1000": 0.0,
     "cost_usd": 0.0,
     "cost_unit": "USD",
     "cost_basis": "ChatGPT subscription",
     "gpu_hours": null,
     "median_latency_ms": 3830.0,
     "mixed_concurrency": false
    },
    {
     "name": "Erk-14B",
     "display_name": "Erk-14B",
     "tag": "off · chat",
     "class_label": "Open · 15B",
     "base_model": null,
     "setup_label": "Erk-14B (chat, reasoning off, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.907535829865927,
     "wilson_low": 0.8945946999691488,
     "wilson_high": 0.9190319167618575,
     "right": 1963,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.35877862595419846,
     "cost_per_1000": 0.2660502388657729,
     "cost_usd": 0.5754666666666668,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.2311111111111111,
     "median_latency_ms": 7890.0,
     "mixed_concurrency": false
    },
    {
     "name": "Raon-1 nano",
     "display_name": "Raon-1 nano",
     "tag": "off · chat",
     "class_label": "Open · 3.1B",
     "base_model": "Raon-1 mini preview (MoE-to-Dense pruning and distillation)",
     "setup_label": "Raon-1 nano (chat, thinking off, KRAFTON vLLM fork, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.8955154877484974,
     "wilson_low": 0.8819155253853106,
     "wilson_high": 0.9077130289123363,
     "right": 1937,
     "n": 2163,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.22058823529411764,
     "cost_per_1000": 1.9007243026660505,
     "cost_usd": 4.111266666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 1.6511111111111112,
     "median_latency_ms": 25550.0,
     "mixed_concurrency": false
    },
    {
     "name": "Karpo Decision 31B",
     "display_name": "Karpo Decision 31B",
     "tag": "",
     "class_label": "Open · 31B",
     "base_model": null,
     "setup_label": "karpo-decision-31B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.8155339805825242,
     "wilson_low": 0.798633655489703,
     "wilson_high": 0.8313154833859767,
     "right": 1764,
     "n": 2163,
     "decision_score": 85.98887595556322,
     "chosen_answer_brier": 0.09575646147125995,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.9025639495564307,
     "non_hallucination": 0.04784688995215311,
     "cost_per_1000": 0.1125597164432116,
     "cost_usd": 0.24346666666666666,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.09777777777777778,
     "median_latency_ms": 154.97186699940357,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-31B-Gen2",
     "display_name": "KnowLine-31B-Gen2",
     "tag": "",
     "class_label": "Open · 31B",
     "base_model": "Gemma 4 31B",
     "setup_label": "KnowLine-31B-Gen2",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.8025889967637541,
     "wilson_low": 0.7852839233361387,
     "wilson_high": 0.8188211483036157,
     "right": 1736,
     "n": 2163,
     "decision_score": 85.04836491793039,
     "chosen_answer_brier": 0.10162169840514643,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8981021811157038,
     "non_hallucination": 0.002325581395348837,
     "cost_per_1000": 0.08314069964555403,
     "cost_usd": 0.17983333333333335,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.07222222222222222,
     "median_latency_ms": 114.2761429946404,
     "mixed_concurrency": false
    },
    {
     "name": "REAX 1.3",
     "display_name": "REAX 1.3",
     "tag": "",
     "class_label": "Open · 57B",
     "base_model": null,
     "setup_label": "REAX 1.3",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.7799352750809061,
     "wilson_low": 0.7619878818406653,
     "wilson_high": 0.7968900721362195,
     "right": 1687,
     "n": 2163,
     "decision_score": 83.57426814851652,
     "chosen_answer_brier": 0.10844991211057575,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.9028980886960594,
     "non_hallucination": 0.023157894736842106,
     "cost_per_1000": 0.14197873324086918,
     "cost_usd": 0.30710000000000004,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.12333333333333334,
     "median_latency_ms": 189.45703300005334,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-31B-Gen1.5",
     "display_name": "KnowLine-31B-Gen1.5",
     "tag": "",
     "class_label": "Open · 31B",
     "base_model": null,
     "setup_label": "KnowLine 31B Gen1.5",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.7739251040221914,
     "wilson_low": 0.7558203839178693,
     "wilson_high": 0.7910585388423362,
     "right": 1674,
     "n": 2163,
     "decision_score": 83.18321528489642,
     "chosen_answer_brier": 0.11026079832426307,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8945987348916302,
     "non_hallucination": 0.010615711252653927,
     "cost_per_1000": 0.04687195917901596,
     "cost_usd": 0.10138404770421153,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.47638888888888886,
     "median_latency_ms": 764.4314320059493,
     "mixed_concurrency": false
    },
    {
     "name": "Copernicus-31B",
     "display_name": "Copernicus-31B",
     "tag": "",
     "class_label": "Open · 31B",
     "base_model": null,
     "setup_label": "Copernicus-31B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.7734627831715211,
     "wilson_low": 0.7553461774654098,
     "wilson_high": 0.7906097428935126,
     "right": 1673,
     "n": 2163,
     "decision_score": 83.06495107808837,
     "chosen_answer_brier": 0.11216376160975355,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8988484574941752,
     "non_hallucination": 0.017204301075268817,
     "cost_per_1000": 0.08441978733240871,
     "cost_usd": 0.1826,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.07333333333333333,
     "median_latency_ms": 117.95815200457582,
     "mixed_concurrency": false
    },
    {
     "name": "Sieve 31B v3",
     "display_name": "Sieve 31B v3",
     "tag": "",
     "class_label": "Open · 31B",
     "base_model": null,
     "setup_label": "Sieve 31B v3",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.7711511789181692,
     "wilson_low": 0.7529756026971353,
     "wilson_high": 0.7883653056553719,
     "right": 1668,
     "n": 2163,
     "decision_score": 82.89465517984281,
     "chosen_answer_brier": 0.11325807532131299,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8910289949858295,
     "non_hallucination": 0.021052631578947368,
     "cost_per_1000": 0.03913740264976726,
     "cost_usd": 0.0846542019314466,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.3977777777777778,
     "median_latency_ms": 637.0208130101673,
     "mixed_concurrency": false
    },
    {
     "name": "Logits Sigma-1 Duo",
     "display_name": "Logits Sigma-1 Duo",
     "tag": "",
     "class_label": "Open · 57B",
     "base_model": null,
     "setup_label": "Logits Sigma-1 Duo",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.7521960240406842,
     "wilson_low": 0.7335647356914493,
     "wilson_high": 0.7699330742084528,
     "right": 1627,
     "n": 2163,
     "decision_score": 81.60426511828958,
     "chosen_answer_brier": 0.12011072167489284,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8987182251006798,
     "non_hallucination": 0.020522388059701493,
     "cost_per_1000": 0.1406996455540145,
     "cost_usd": 0.30433333333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.12222222222222222,
     "median_latency_ms": 196.46159900003113,
     "mixed_concurrency": false
    },
    {
     "name": "d1-ultra",
     "display_name": "d1-ultra",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Liquid d1-ultra",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.7411003236245954,
     "wilson_low": 0.7222242789989598,
     "wilson_high": 0.7591214732701493,
     "right": 1603,
     "n": 2163,
     "decision_score": 80.87791700454547,
     "chosen_answer_brier": 0.12354198353368612,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8762476606363069,
     "non_hallucination": 0.8740894901144641,
     "cost_per_1000": 0.0061411456310679615,
     "cost_usd": 0.013283298,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 179.22829499002546,
     "mixed_concurrency": false
    },
    {
     "name": "Mammoth-1",
     "display_name": "Mammoth-1",
     "tag": "",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Mammoth-1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.7397133610725843,
     "wilson_low": 0.7208078168855339,
     "wilson_high": 0.757768928179726,
     "right": 1600,
     "n": 2163,
     "decision_score": 81.16364901712828,
     "chosen_answer_brier": 0.11644038073001896,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8947291296625222,
     "non_hallucination": 0.7057256990679095,
     "cost_per_1000": 0.022563430420711975,
     "cost_usd": 0.048804700000000006,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.09027777777777778,
     "median_latency_ms": 140.25158399999782,
     "mixed_concurrency": false
    },
    {
     "name": "Perplexity Decider 27B v1.3",
     "display_name": "Perplexity Decider 27B v1.3",
     "tag": "NVIDIA RTX PRO 6000 Blackwell",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Perplexity Decider 27B v1.3 (local GPU)",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.7249190938511327,
     "wilson_low": 0.7057135246203687,
     "wilson_high": 0.7433271436038338,
     "right": 1568,
     "n": 2163,
     "decision_score": 79.32218663069054,
     "chosen_answer_brier": 0.13847536123732193,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8558191133596296,
     "non_hallucination": 0.09433962264150944,
     "cost_per_1000": 0.06587301587301587,
     "cost_usd": 0.14248333333333335,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.05722222222222222,
     "median_latency_ms": 91.26321799885773,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE 27B",
     "display_name": "GLiDE 27B",
     "tag": "",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "GLiDE 27B (open)",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.7198335644937587,
     "wilson_low": 0.7005308906504427,
     "wilson_high": 0.7383567511596462,
     "right": 1557,
     "n": 2163,
     "decision_score": 79.21924223892869,
     "chosen_answer_brier": 0.13544871971518474,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8637474537434475,
     "non_hallucination": 0.21638330757341576,
     "cost_per_1000": 0.026155373139544184,
     "cost_usd": 0.05657407210083407,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.2658333333333333,
     "median_latency_ms": 425.01643899595365,
     "mixed_concurrency": false
    },
    {
     "name": "d3-ultra",
     "display_name": "d3-ultra",
     "tag": "",
     "class_label": "Open · 26B",
     "base_model": null,
     "setup_label": "vLLM-SR d3-ultra",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.7152103559870551,
     "wilson_low": 0.6958220088904515,
     "wilson_high": 0.7338356089068071,
     "right": 1547,
     "n": 2163,
     "decision_score": 78.39226911826918,
     "chosen_answer_brier": 0.1473649736216714,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8571816838623562,
     "non_hallucination": 0.05345911949685535,
     "cost_per_1000": 0.08857682231468639,
     "cost_usd": 0.19159166666666666,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.07694444444444444,
     "median_latency_ms": 118.26817499968456,
     "mixed_concurrency": false
    },
    {
     "name": "Gemini Decision 1",
     "display_name": "Gemini Decision 1",
     "tag": "20261111",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Gemini Decision 1",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.7133610725843735,
     "wilson_low": 0.6939391419021838,
     "wilson_high": 0.7320264662899424,
     "right": 1543,
     "n": 2163,
     "decision_score": 78.83578263297753,
     "chosen_answer_brier": 0.13664541992482293,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8590209687872389,
     "non_hallucination": 0.038980509745127435,
     "cost_per_1000": 0.022024641701340733,
     "cost_usd": 0.0476393,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 226.15016798954457,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE",
     "display_name": "GLiDE",
     "tag": "API",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Fastino GLiDE",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.7078132223763292,
     "wilson_low": 0.6882928625888002,
     "wilson_high": 0.7265967167879296,
     "right": 1531,
     "n": 2163,
     "decision_score": 78.55622249056535,
     "chosen_answer_brier": 0.13668877256502204,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8654949606859089,
     "non_hallucination": 0.40554821664464996,
     "cost_per_1000": 0.02829514563106796,
     "cost_usd": 0.0612024,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 264.3573050154373,
     "mixed_concurrency": false
    },
    {
     "name": "d3",
     "display_name": "d3",
     "tag": "",
     "class_label": "Open · 26B",
     "base_model": null,
     "setup_label": "vLLM-SR d3",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.7041146555709662,
     "wilson_low": 0.6845305893541189,
     "wilson_high": 0.7229749708123463,
     "right": 1523,
     "n": 2163,
     "decision_score": 78.70869817917844,
     "chosen_answer_brier": 0.12994069198739724,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8783978988837821,
     "non_hallucination": 0.047400611620795105,
     "cost_per_1000": 0.040611034057636,
     "cost_usd": 0.08784166666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.035277777777777776,
     "median_latency_ms": 55.095481000080326,
     "mixed_concurrency": false
    },
    {
     "name": "Drex Max",
     "display_name": "Drex Max",
     "tag": "",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Drex Max v1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.7031900138696255,
     "wilson_low": 0.683590257749352,
     "wilson_high": 0.7220692976145473,
     "right": 1521,
     "n": 2163,
     "decision_score": 77.10428309608832,
     "chosen_answer_brier": 0.16110435194785896,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8612048148352965,
     "non_hallucination": 0.044753086419753084,
     "cost_per_1000": 0.013058058252427184,
     "cost_usd": 0.02824458,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.0475,
     "median_latency_ms": 75.94410399906337,
     "mixed_concurrency": false
    },
    {
     "name": "Drex Max v1.1",
     "display_name": "Drex Max v1.1",
     "tag": "",
     "class_label": "Closed · Efficient (provisional)",
     "base_model": null,
     "setup_label": "Drex Max 1.1",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.7008784096162737,
     "wilson_low": 0.6812398401715052,
     "wilson_high": 0.7198047031859789,
     "right": 1516,
     "n": 2163,
     "decision_score": 78.00980527580843,
     "chosen_answer_brier": 0.14068230410010515,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8667658321540864,
     "non_hallucination": 0.43483870967741933,
     "cost_per_1000": 0.012878058252427183,
     "cost_usd": 0.027855239999999996,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 326.3873489340767,
     "mixed_concurrency": false
    },
    {
     "name": "celeris-1-decision",
     "display_name": "celeris-1-decision",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Celeris-1 Decision",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.6999537679149329,
     "wilson_low": 0.6802998370553325,
     "wilson_high": 0.7188987014995852,
     "right": 1514,
     "n": 2163,
     "decision_score": 77.96068423485899,
     "chosen_answer_brier": 0.1407400832177531,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8577732636125489,
     "non_hallucination": 0.8268268268268268,
     "cost_per_1000": 0.009096643550624135,
     "cost_usd": 0.019676040000000002,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 387.0343810413033,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE",
     "display_name": "GLiDE",
     "tag": "v1.2",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Fastino GLiDE v1.2",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.6985668053629218,
     "wilson_low": 0.678890007230676,
     "wilson_high": 0.7175395241203926,
     "right": 1511,
     "n": 2163,
     "decision_score": 77.94871134503826,
     "chosen_answer_brier": 0.13959257846215656,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8624879716435303,
     "non_hallucination": 0.3578231292517007,
     "cost_per_1000": 0.0075453721682847905,
     "cost_usd": 0.01632064,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.050277777777777775,
     "median_latency_ms": 78.70889001060277,
     "mixed_concurrency": false
    },
    {
     "name": "celeris-1.1-decision",
     "display_name": "celeris-1.1-decision",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Celeris-1.1 Decision",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.6985668053629218,
     "wilson_low": 0.678890007230676,
     "wilson_high": 0.7175395241203926,
     "right": 1511,
     "n": 2163,
     "decision_score": 77.91241562644475,
     "chosen_answer_brier": 0.1403184928340268,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8593189818630655,
     "non_hallucination": 0.8291203235591507,
     "cost_per_1000": 0.0009096643550624133,
     "cost_usd": 0.001967604,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 381.0945700024604,
     "mixed_concurrency": false
    },
    {
     "name": "d1.2-ultra",
     "display_name": "d1.2-ultra",
     "tag": "",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "Liquid d1.2-ultra",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.6985668053629218,
     "wilson_low": 0.678890007230676,
     "wilson_high": 0.7175395241203926,
     "right": 1511,
     "n": 2163,
     "decision_score": 78.22911759607146,
     "chosen_answer_brier": 0.13398445344149273,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8759749566573146,
     "non_hallucination": 0.866923818707811,
     "cost_per_1000": 0.0061411456310679615,
     "cost_usd": 0.013283298,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 214.74233000117238,
     "mixed_concurrency": false
    },
    {
     "name": "Perplexity Decider 27B v1.2",
     "display_name": "Perplexity Decider 27B v1.2",
     "tag": "API",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Perplexity Decider 27B v1.2 (API)",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.6976421636615812,
     "wilson_low": 0.6779502368911594,
     "wilson_high": 0.7166332896573433,
     "right": 1509,
     "n": 2163,
     "decision_score": 78.15651640241289,
     "chosen_answer_brier": 0.1345118356133232,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8691186216037111,
     "non_hallucination": 0.06983655274888559,
     "cost_per_1000": 0.004292686084142394,
     "cost_usd": 0.00928508,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 226.48629600007553,
     "mixed_concurrency": false
    },
    {
     "name": "Perplexity Decider 27B v1.2",
     "display_name": "Perplexity Decider 27B v1.2",
     "tag": "API · with web",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Perplexity Decider 27B v1.2 (API, with web)",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.6930189551548774,
     "wilson_low": 0.6732527707316596,
     "wilson_high": 0.7121007318040125,
     "right": 1499,
     "n": 2163,
     "decision_score": 77.9400708286922,
     "chosen_answer_brier": 0.1342175385810335,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8733121277638908,
     "non_hallucination": 0.15350223546944858,
     "cost_per_1000": 1.0227930651872401,
     "cost_usd": 2.2123014000000003,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 421.51945299974614,
     "mixed_concurrency": false
    },
    {
     "name": "Turion Jev 31B",
     "display_name": "Turion Jev 31B",
     "tag": "",
     "class_label": "Open · 31B",
     "base_model": null,
     "setup_label": "Turion Jev 31B",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.6902450300508552,
     "wilson_low": 0.6704353910416745,
     "wilson_high": 0.7093800970862993,
     "right": 1493,
     "n": 2163,
     "decision_score": 77.51885105579798,
     "chosen_answer_brier": 0.1398680089348955,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8643810418770181,
     "non_hallucination": 0.021806853582554516,
     "cost_per_1000": 0.017464246014805365,
     "cost_usd": 0.037775164130024005,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.1775,
     "median_latency_ms": 278.4201280446723,
     "mixed_concurrency": false
    },
    {
     "name": "SubQ Snap 1.1",
     "display_name": "SubQ Snap 1.1",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "SubQ Snap 1.1",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.6856218215441516,
     "wilson_low": 0.6657415719120381,
     "wilson_high": 0.7048438922031053,
     "right": 1483,
     "n": 2163,
     "decision_score": 76.15183773819994,
     "chosen_answer_brier": 0.16258506678015267,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8391148704930388,
     "non_hallucination": 0.5248815165876777,
     "cost_per_1000": 0.008585372168284788,
     "cost_usd": 0.01857016,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 380.88442711159587,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-12B-Gen1",
     "display_name": "KnowLine-12B-Gen1",
     "tag": "",
     "class_label": "Open · 12B",
     "base_model": "Gemma 4 12B",
     "setup_label": "KnowLine-12B-Gen1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.6625057790106333,
     "wilson_low": 0.6423056226916681,
     "wilson_high": 0.6821297213593229,
     "right": 1433,
     "n": 2163,
     "decision_score": 74.9643473123793,
     "chosen_answer_brier": 0.16321883276304727,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8193740500339359,
     "non_hallucination": 0.01092896174863388,
     "cost_per_1000": 0.03613422715364463,
     "cost_usd": 0.07815833333333333,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.03138888888888889,
     "median_latency_ms": 49.042962986277416,
     "mixed_concurrency": false
    },
    {
     "name": "d1.2",
     "display_name": "d1.2",
     "tag": "",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "Liquid d1.2",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.6551086453999075,
     "wilson_low": 0.6348174681959393,
     "wilson_high": 0.6748498374345231,
     "right": 1417,
     "n": 2163,
     "decision_score": 74.97541289354976,
     "chosen_answer_brier": 0.15560038752891212,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8378257315894131,
     "non_hallucination": 0.5627753303964758,
     "cost_per_1000": 0.002924355062413315,
     "cost_usd": 0.00632538,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 203.9076639994164,
     "mixed_concurrency": false
    },
    {
     "name": "d1.1",
     "display_name": "d1.1",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Liquid d1.1",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.651872399445215,
     "wilson_low": 0.6315430884980227,
     "wilson_high": 0.6716632003234584,
     "right": 1410,
     "n": 2163,
     "decision_score": 74.61851013292811,
     "chosen_answer_brier": 0.15950219678665295,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8292871068915826,
     "non_hallucination": 0.8252595155709342,
     "cost_per_1000": 0.002924355062413315,
     "cost_usd": 0.00632538,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 171.83256801217794,
     "mixed_concurrency": false
    },
    {
     "name": "Jev 1.14",
     "display_name": "Jev 1.14",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Jev 1.14.0",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.6514100785945446,
     "wilson_low": 0.6310754032036784,
     "wilson_high": 0.6712078832165195,
     "right": 1409,
     "n": 2163,
     "decision_score": 74.56213823393435,
     "chosen_answer_brier": 0.1601673139158576,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8290202431131435,
     "non_hallucination": 0.07722007722007722,
     "cost_per_1000": 0.018993922330097085,
     "cost_usd": 0.041083853999999996,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 163.25783799402416,
     "mixed_concurrency": false
    },
    {
     "name": "Jev 1.13",
     "display_name": "Jev 1.13",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Jev 1.13.0",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.6407766990291263,
     "wilson_low": 0.6203243308109062,
     "wilson_high": 0.6607299003797819,
     "right": 1386,
     "n": 2163,
     "decision_score": 74.07820388349515,
     "chosen_answer_brier": 0.1592126213592233,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8335979764551193,
     "non_hallucination": 0.05674653215636822,
     "cost_per_1000": 0.018993922330097085,
     "cost_usd": 0.041083853999999996,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 166.1319190170616,
     "mixed_concurrency": false
    },
    {
     "name": "Adjudex 9B v5",
     "display_name": "Adjudex 9B v5",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": "Qwen3.5-9B",
     "setup_label": "Adjudex 9B v5",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.6208969024503005,
     "wilson_low": 0.6002532018762495,
     "wilson_high": 0.6411119260592674,
     "right": 1343,
     "n": 2163,
     "decision_score": 72.28416679725225,
     "chosen_answer_brier": 0.17521356650525546,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8064934711149048,
     "non_hallucination": 0.030878859857482184,
     "cost_per_1000": 0.026541069502234553,
     "cost_usd": 0.05740833333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.023055555555555555,
     "median_latency_ms": 35.929751000367105,
     "mixed_concurrency": false
    },
    {
     "name": "Elephant-1",
     "display_name": "Elephant-1",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": null,
     "setup_label": "Elephant-1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.6134997688395747,
     "wilson_low": 0.592794219422916,
     "wilson_high": 0.6338028700920723,
     "right": 1327,
     "n": 2163,
     "decision_score": 71.85909303325175,
     "chosen_answer_brier": 0.17631790817453952,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8031818902946892,
     "non_hallucination": 0.649908592321755,
     "cost_per_1000": 0.00789720064724919,
     "cost_usd": 0.017081645,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.050555555555555555,
     "median_latency_ms": 80.30326300013257,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-9B-Gen2",
     "display_name": "KnowLine-9B-Gen2",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": "Qwen3.5-9B",
     "setup_label": "KnowLine-9B-Gen2",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.6010171058714748,
     "wilson_low": 0.580218450817758,
     "wilson_high": 0.6214575738625879,
     "right": 1300,
     "n": 2163,
     "decision_score": 71.15404664899832,
     "chosen_answer_brier": 0.17793617289150834,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.803133077814422,
     "non_hallucination": 0.032036613272311214,
     "cost_per_1000": 0.03101787640622592,
     "cost_usd": 0.06709166666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.026944444444444444,
     "median_latency_ms": 40.8647499862127,
     "mixed_concurrency": false
    },
    {
     "name": "Jeffan-9B-1211",
     "display_name": "Jeffan-9B-1211",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": "Qwen3.5-9B",
     "setup_label": "Jeffan-9B-1211",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5982431807674526,
     "wilson_low": 0.5774257369301428,
     "wilson_high": 0.6187122733425048,
     "right": 1294,
     "n": 2163,
     "decision_score": 71.31556221615922,
     "chosen_answer_brier": 0.17193193644426794,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8139412140302325,
     "non_hallucination": 0.06614349775784753,
     "cost_per_1000": 0.02877947295423024,
     "cost_usd": 0.06225000000000001,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.025,
     "median_latency_ms": 36.87551499751862,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-9B-Gen1",
     "display_name": "KnowLine-9B-Gen1",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": "Qwen3.5-9B",
     "setup_label": "KnowLine-9B-Gen1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5977808599167822,
     "wilson_low": 0.5769603513840923,
     "wilson_high": 0.6182546564872721,
     "right": 1293,
     "n": 2163,
     "decision_score": 71.07883643998467,
     "chosen_answer_brier": 0.17620413111708877,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.7988132383924047,
     "non_hallucination": 0.009280742459396751,
     "cost_per_1000": 0.026541069502234553,
     "cost_usd": 0.05740833333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.023055555555555555,
     "median_latency_ms": 36.90840899798786,
     "mixed_concurrency": false
    },
    {
     "name": "Microsoft-Decision-1",
     "display_name": "Microsoft-Decision-1",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": "Qwen3.5-9B",
     "setup_label": "Microsoft-Decision-1",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.5936199722607489,
     "wilson_low": 0.5727727373522037,
     "wilson_high": 0.6141352489076134,
     "right": 1284,
     "n": 2163,
     "decision_score": 70.48752097210051,
     "chosen_answer_brier": 0.18386955281873868,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.7872121746958275,
     "non_hallucination": 0.050605060506050605,
     "cost_per_1000": 0.006807533980582525,
     "cost_usd": 0.014724696000000002,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 310.0611539557576,
     "mixed_concurrency": false
    },
    {
     "name": "Drex 2.0",
     "display_name": "Drex 2.0",
     "tag": "",
     "class_label": "Open · 35B MoE · 3.0B active",
     "base_model": null,
     "setup_label": "Drex 2.0",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.5899214054553861,
     "wilson_low": 0.5690517028799554,
     "wilson_high": 0.6104722641695973,
     "right": 1276,
     "n": 2163,
     "decision_score": 70.9813936243643,
     "chosen_answer_brier": 0.17029353296809985,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8125784140829042,
     "non_hallucination": 0.2074688796680498,
     "cost_per_1000": 0.008918058252427185,
     "cost_usd": 0.01928976,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 334.1465060000246,
     "mixed_concurrency": false
    },
    {
     "name": "Dexy-Decision-9B-V2",
     "display_name": "Dexy-Decision-9B-V2",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": "vllm-sr/d3-flash",
     "setup_label": "Dexy-Decision-9B-V2",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5880721220527045,
     "wilson_low": 0.5671916389408139,
     "wilson_high": 0.6086403185036066,
     "right": 1272,
     "n": 2163,
     "decision_score": 69.67350528523042,
     "chosen_answer_brier": 0.19460201634809626,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.8035817645356429,
     "non_hallucination": 0.016666666666666666,
     "cost_per_1000": 0.03197719217136693,
     "cost_usd": 0.06916666666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.027777777777777776,
     "median_latency_ms": 45.31979800958652,
     "mixed_concurrency": false
    },
    {
     "name": "StartLux-Decision-v1.1-9B-Instant",
     "display_name": "StartLux-Decision-v1.1-9B-Instant",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": null,
     "setup_label": "StartLux-Decision v1.1 9B",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.5843735552473417,
     "wilson_low": 0.5634724145854182,
     "wilson_high": 0.604975523648738,
     "right": 1264,
     "n": 2163,
     "decision_score": 69.89653901312944,
     "chosen_answer_brier": 0.18644277498475273,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.7811465974852508,
     "non_hallucination": 0.04220779220779221,
     "cost_per_1000": 0.008335829474985346,
     "cost_usd": 0.018030399154393303,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.08472222222222223,
     "median_latency_ms": 140.37829398876056,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE 9B",
     "display_name": "GLiDE 9B",
     "tag": "",
     "class_label": "Open · 7.9B",
     "base_model": null,
     "setup_label": "GLiDE 9B (open)",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.5792880258899676,
     "wilson_low": 0.5583604408090972,
     "wilson_high": 0.5999344710109453,
     "right": 1253,
     "n": 2163,
     "decision_score": 69.68961977603017,
     "chosen_answer_brier": 0.18549563036936448,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.7979048086789507,
     "non_hallucination": 0.24575424575424576,
     "cost_per_1000": 0.010494945961948764,
     "cost_usd": 0.022700568115695176,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.10666666666666667,
     "median_latency_ms": 174.3119540042244,
     "mixed_concurrency": false
    },
    {
     "name": "vera-medium-omni",
     "display_name": "vera-medium-omni",
     "tag": "",
     "class_label": "Open · 4.2B",
     "base_model": "Qwen3.5-4B",
     "setup_label": "vera-medium-omni",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5681923254738789,
     "wilson_low": 0.5472148687043519,
     "wilson_high": 0.5889279854848977,
     "right": 1229,
     "n": 2163,
     "decision_score": 69.17730056842348,
     "chosen_answer_brier": 0.18464631410540913,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.7891942231197174,
     "non_hallucination": 0.03825717321997875,
     "cost_per_1000": 0.02334335028509786,
     "cost_usd": 0.05049166666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.020277777777777777,
     "median_latency_ms": 31.016872002510354,
     "mixed_concurrency": false
    },
    {
     "name": "Cerno 9B v1.2",
     "display_name": "Cerno 9B v1.2",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": null,
     "setup_label": "Cerno 9B v1.2",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.5603328710124826,
     "wilson_low": 0.5393265351760993,
     "wilson_high": 0.5811252781913384,
     "right": 1212,
     "n": 2163,
     "decision_score": 67.14801432269995,
     "chosen_answer_brier": 0.2173725845584836,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.7799376546487456,
     "non_hallucination": 0.011879049676025918,
     "cost_per_1000": 0.009511044778016068,
     "cost_usd": 0.020572389854848754,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.09666666666666666,
     "median_latency_ms": 152.799007948488,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine 4B Gen5.5",
     "display_name": "KnowLine 4B Gen5.5",
     "tag": "",
     "class_label": "Open · 4.0B",
     "base_model": null,
     "setup_label": "KnowLine 4B Gen5.5",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.5598705501618123,
     "wilson_low": 0.5388626811648223,
     "wilson_high": 0.5806661298013323,
     "right": 1211,
     "n": 2163,
     "decision_score": 67.88763827422379,
     "chosen_answer_brier": 0.20211778467733627,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.765447508483162,
     "non_hallucination": 0.01644398766700925,
     "cost_per_1000": 0.0045642082699100086,
     "cost_usd": 0.00987238248781535,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.04638888888888889,
     "median_latency_ms": 74.07237499137409,
     "mixed_concurrency": false
    },
    {
     "name": "Perplexity Decider Flash v1.3",
     "display_name": "Perplexity Decider Flash v1.3",
     "tag": "",
     "class_label": "Open · 4.0B",
     "base_model": null,
     "setup_label": "Perplexity Decider Flash v1.3",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.559408229311142,
     "wilson_low": 0.5383988455138565,
     "wilson_high": 0.580206963051015,
     "right": 1210,
     "n": 2163,
     "decision_score": 67.89499568939264,
     "chosen_answer_brier": 0.20150831552328913,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.7874498105157267,
     "non_hallucination": 0.08401639344262295,
     "cost_per_1000": 0.022384034519956852,
     "cost_usd": 0.04841666666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.019444444444444445,
     "median_latency_ms": 28.229934998307726,
     "mixed_concurrency": false
    },
    {
     "name": "StartLux-Decision-v1.1-4B-Instant",
     "display_name": "StartLux-Decision-v1.1-4B-Instant",
     "tag": "",
     "class_label": "Open · 4.0B",
     "base_model": null,
     "setup_label": "StartLux-Decision v1.1 4B",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.5584835876098012,
     "wilson_low": 0.5374712292683075,
     "wilson_high": 0.579288574493998,
     "right": 1208,
     "n": 2163,
     "decision_score": 68.25256158069982,
     "chosen_answer_brier": 0.19343235599580497,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.7788486876321903,
     "non_hallucination": 0.13391136801541426,
     "cost_per_1000": 0.004099588266386236,
     "cost_usd": 0.008867409420193428,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.041666666666666664,
     "median_latency_ms": 66.50415097828954,
     "mixed_concurrency": false
    },
    {
     "name": "d1",
     "display_name": "d1",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Liquid d1",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.5561719833564494,
     "wilson_low": 0.5351525095355927,
     "wilson_high": 0.5769922822202975,
     "right": 1203,
     "n": 2163,
     "decision_score": 68.1603077282477,
     "chosen_answer_brier": 0.1929658287914955,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.7668731816292602,
     "non_hallucination": 0.028942115768463075,
     "cost_per_1000": 0.002924355062413315,
     "cost_usd": 0.00632538,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 183.52204689290375,
     "mixed_concurrency": false
    },
    {
     "name": "Valen-Decision-1.5-9B",
     "display_name": "Valen-Decision-1.5-9B",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": "Qwen3.5-9B",
     "setup_label": "Valen-Decision-1.5-9B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5533980582524272,
     "wilson_low": 0.5323706501106228,
     "wilson_high": 0.5742361272375692,
     "right": 1197,
     "n": 2163,
     "decision_score": 68.36949738237469,
     "chosen_answer_brier": 0.1860081106049336,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.7901292222965972,
     "non_hallucination": 0.026422764227642278,
     "cost_per_1000": 0.04093080597934967,
     "cost_usd": 0.08853333333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.035555555555555556,
     "median_latency_ms": 58.23571099608671,
     "mixed_concurrency": false
    },
    {
     "name": "Perplexity Decider Flash v1.2",
     "display_name": "Perplexity Decider Flash v1.2",
     "tag": "",
     "class_label": "Open · 4.0B",
     "base_model": null,
     "setup_label": "Perplexity Decider Flash v1.2",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5524734165510865,
     "wilson_low": 0.5314435099254424,
     "wilson_high": 0.5733172626201835,
     "right": 1195,
     "n": 2163,
     "decision_score": 67.61161196382137,
     "chosen_answer_brier": 0.20024117727465893,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.7808162453750129,
     "non_hallucination": 0.11976630963972736,
     "cost_per_1000": 0.0021763430420711975,
     "cost_usd": 0.00470743,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.026111111111111113,
     "median_latency_ms": 39.528700057417154,
     "mixed_concurrency": false
    },
    {
     "name": "Open-Jev-27B-v1.1",
     "display_name": "Open-Jev-27B-v1.1",
     "tag": "",
     "class_label": "Open · 27B",
     "base_model": "Qwen3.8-27B + adapter",
     "setup_label": "Open-Jev 27B v1.1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.545538603791031,
     "wilson_low": 0.5244922848439917,
     "wilson_high": 0.5664234516823885,
     "right": 1180,
     "n": 2163,
     "decision_score": 64.53129531330411,
     "chosen_answer_brier": 0.2549126975249486,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.7635455282169767,
     "non_hallucination": 0.049554013875123884,
     "cost_per_1000": 0.16915934658653103,
     "cost_usd": 0.36589166666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.14694444444444443,
     "median_latency_ms": 233.04351499973563,
     "mixed_concurrency": false
    },
    {
     "name": "OneJev-27B",
     "display_name": "OneJev-27B",
     "tag": "",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "OneJev 27B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5376791493296348,
     "wilson_low": 0.5166191777083311,
     "wilson_high": 0.5586055179962373,
     "right": 1163,
     "n": 2163,
     "decision_score": 67.53705215239825,
     "chosen_answer_brier": 0.1869381062816699,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.7834187446259673,
     "non_hallucination": 0.01794616151545364,
     "cost_per_1000": 0.06075666512559717,
     "cost_usd": 0.13141666666666668,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.05277777777777778,
     "median_latency_ms": 79.32667699969898,
     "mixed_concurrency": false
    },
    {
     "name": "Mercury Decide",
     "display_name": "Mercury Decide",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Inception Mercury Decide",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.5349052242256126,
     "wilson_low": 0.5138416854151927,
     "wilson_high": 0.5558449958816775,
     "right": 1157,
     "n": 2163,
     "decision_score": 66.0166025866824,
     "chosen_answer_brier": 0.2145731724919646,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.7892253222239596,
     "non_hallucination": 0.031777557100297914,
     "cost_per_1000": 0.004384336569579287,
     "cost_usd": 0.00948332,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 287.7879429142922,
     "mixed_concurrency": false
    },
    {
     "name": "Verdin-4B-v1.1",
     "display_name": "Verdin-4B-v1.1",
     "tag": "",
     "class_label": "Open · 4.0B",
     "base_model": "Qwen3.5-4B",
     "setup_label": "Verdin-4B-v1.1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5150254276467868,
     "wilson_low": 0.4939553528478791,
     "wilson_high": 0.5360422251938199,
     "right": 1114,
     "n": 2163,
     "decision_score": 65.62151070613683,
     "chosen_answer_brier": 0.20259521352405022,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.7516280359340263,
     "non_hallucination": 0.06433823529411764,
     "cost_per_1000": 0.03357605177993528,
     "cost_usd": 0.07262500000000001,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.029166666666666667,
     "median_latency_ms": 47.14914700889494,
     "mixed_concurrency": false
    },
    {
     "name": "Clef",
     "display_name": "Clef",
     "tag": "",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Clef",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5108645399907535,
     "wilson_low": 0.48979731266540855,
     "wilson_high": 0.531893243764743,
     "right": 1105,
     "n": 2163,
     "decision_score": 65.19551591400833,
     "chosen_answer_brier": 0.20695422171058714,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.7670820894884055,
     "non_hallucination": 0.016205910390848427,
     "cost_per_1000": 0.06854024965325937,
     "cost_usd": 0.14825256,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.11416666666666667,
     "median_latency_ms": 180.95704999996087,
     "mixed_concurrency": false
    },
    {
     "name": "Chinese Decision Pointer 4B",
     "display_name": "Chinese Decision Pointer 4B",
     "tag": "",
     "class_label": "Open · 4.0B",
     "base_model": "Qwen3.5-4B",
     "setup_label": "Chinese Decision Pointer 4B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5043920480813685,
     "wilson_low": 0.4833321447838583,
     "wilson_high": 0.5254363780283307,
     "right": 1091,
     "n": 2163,
     "decision_score": 64.1143792402495,
     "chosen_answer_brier": 0.2221044632763784,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.7460326689193811,
     "non_hallucination": 0.034195933456561925,
     "cost_per_1000": 0.02270380644167052,
     "cost_usd": 0.04910833333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.01972222222222222,
     "median_latency_ms": 28.425094002159312,
     "mixed_concurrency": false
    },
    {
     "name": "Clef-Flash",
     "display_name": "Clef-Flash",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": null,
     "setup_label": "Clef-flash",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.49791955617198336,
     "wilson_low": 0.47687049909588924,
     "wilson_high": 0.5189759900983371,
     "right": 1077,
     "n": 2163,
     "decision_score": 64.43759090661119,
     "chosen_answer_brier": 0.20916773803975958,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.729690446999116,
     "non_hallucination": 0.01981981981981982,
     "cost_per_1000": 0.010852206195099398,
     "cost_usd": 0.023473322,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.0625,
     "median_latency_ms": 99.37365900077566,
     "mixed_concurrency": false
    },
    {
     "name": "Winnow-12B",
     "display_name": "Winnow-12B",
     "tag": "",
     "class_label": "Open · 12B",
     "base_model": null,
     "setup_label": "Winnow-12B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.46879334257975036,
     "wilson_low": 0.44783671372134554,
     "wilson_high": 0.48986062419204884,
     "right": 1014,
     "n": 2163,
     "decision_score": 63.004975273352585,
     "chosen_answer_brier": 0.20869383711269865,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.7433528512058337,
     "non_hallucination": 0.09196354598177299,
     "cost_per_1000": 0.038372630605640314,
     "cost_usd": 0.083,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.03333333333333333,
     "median_latency_ms": 51.76649500026542,
     "mixed_concurrency": false
    },
    {
     "name": "Cygnet",
     "display_name": "Cygnet",
     "tag": "",
     "class_label": "Open · 12B",
     "base_model": "Gemma 4 12B IT",
     "setup_label": "Cygnet",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.44336569579288027,
     "wilson_low": 0.4225484226765661,
     "wilson_high": 0.4643837831662606,
     "right": 959,
     "n": 2163,
     "decision_score": 61.86765677781185,
     "chosen_answer_brier": 0.20601256023664305,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.734638881864068,
     "non_hallucination": 0.04914004914004914,
     "cost_per_1000": 0.023944105409153957,
     "cost_usd": 0.05179110000000001,
     "cost_unit": "USD",
     "cost_basis": "public host's list price (same weights)",
     "gpu_hours": 0.02388888888888889,
     "median_latency_ms": 36.978988000555546,
     "mixed_concurrency": false
    },
    {
     "name": "Metask-Jev-4B",
     "display_name": "Metask-Jev-4B",
     "tag": "",
     "class_label": "Open · 4.5B",
     "base_model": null,
     "setup_label": "Metask-Jev 4B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.39158576051779936,
     "wilson_low": 0.3712249973689481,
     "wilson_high": 0.4123309395301773,
     "right": 847,
     "n": 2163,
     "decision_score": 58.63013617829513,
     "chosen_answer_brier": 0.21898303695189675,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.6507156493685922,
     "non_hallucination": 0.022222222222222223,
     "cost_per_1000": 0.08314069964555403,
     "cost_usd": 0.17983333333333335,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.07222222222222222,
     "median_latency_ms": 112.5019199680537,
     "mixed_concurrency": false
    },
    {
     "name": "StartLux-Decision-v1.1-2B-Instant",
     "display_name": "StartLux-Decision-v1.1-2B-Instant",
     "tag": "",
     "class_label": "Open · 2.0B",
     "base_model": null,
     "setup_label": "StartLux-Decision v1.1 2B",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.3865002311604253,
     "wilson_low": 0.3661971299079278,
     "wilson_high": 0.40720578057708406,
     "right": 836,
     "n": 2163,
     "decision_score": 58.94816286012234,
     "chosen_answer_brier": 0.20753697395797868,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.684695485373707,
     "non_hallucination": 0.07153284671532846,
     "cost_per_1000": 0.0022684388407337172,
     "cost_usd": 0.00490663321250703,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.023055555555555555,
     "median_latency_ms": 35.87926799082197,
     "mixed_concurrency": false
    },
    {
     "name": "d1-2B",
     "display_name": "d1-2B",
     "tag": "",
     "class_label": "Open · 1.9B",
     "base_model": "LFM3-VL-2B",
     "setup_label": "Liquid d1-2B (open weights)",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.37078132223763294,
     "wilson_low": 0.35067140639808914,
     "wilson_high": 0.3913494224432992,
     "right": 802,
     "n": 2163,
     "decision_score": 57.72471453130048,
     "chosen_answer_brier": 0.21628703161162324,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.6555465670870583,
     "non_hallucination": 0.802013422818792,
     "cost_per_1000": 0.01406996455540145,
     "cost_usd": 0.030433333333333333,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.012222222222222223,
     "median_latency_ms": 16.47641498129815,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE 2B",
     "display_name": "GLiDE 2B",
     "tag": "",
     "class_label": "Open · 1.9B",
     "base_model": null,
     "setup_label": "GLiDE 2B (open)",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.36754507628294036,
     "wilson_low": 0.34747777451003203,
     "wilson_high": 0.3880820375223749,
     "right": 795,
     "n": 2163,
     "decision_score": 55.69827914941818,
     "chosen_answer_brier": 0.2535794932945768,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.6349930118798043,
     "non_hallucination": 0.17287420941672524,
     "cost_per_1000": 0.0017764882487673688,
     "cost_usd": 0.0038425440820838187,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.018055555555555554,
     "median_latency_ms": 27.5637159938924,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-2B-Gen1",
     "display_name": "KnowLine-2B-Gen1",
     "tag": "",
     "class_label": "Open · 2.0B",
     "base_model": "Qwen3.5-2B",
     "setup_label": "KnowLine-2B-Gen1",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.35275080906148865,
     "wilson_low": 0.3328909022433886,
     "wilson_high": 0.37313283294796085,
     "right": 763,
     "n": 2163,
     "decision_score": 56.49454109467982,
     "chosen_answer_brier": 0.22285998716789235,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.635515821007302,
     "non_hallucination": 0.016913319238900635,
     "cost_per_1000": 0.0031976788477812643,
     "cost_usd": 0.006916579347750875,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.0325,
     "median_latency_ms": 49.86440599896014,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE 0.8B",
     "display_name": "GLiDE 0.8B",
     "tag": "",
     "class_label": "Open · 0.8B",
     "base_model": null,
     "setup_label": "GLiDE 0.8B (open)",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.32085067036523346,
     "wilson_low": 0.3015105578322408,
     "wilson_high": 0.34082601167057835,
     "right": 694,
     "n": 2163,
     "decision_score": 53.21072228123194,
     "chosen_answer_brier": 0.2566362247405946,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.5891920045983957,
     "non_hallucination": 0.20830670926517572,
     "cost_per_1000": 0.0012845376568010206,
     "cost_usd": 0.0027784549516606076,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.013055555555555556,
     "median_latency_ms": 19.813018006971106,
     "mixed_concurrency": false
    },
    {
     "name": "StartLux-Decision-v1.1-0.8B-Instant",
     "display_name": "StartLux-Decision-v1.1-0.8B-Instant",
     "tag": "",
     "class_label": "Open · 0.8B",
     "base_model": "Qwen3.5-0.8B",
     "setup_label": "StartLux-Decision-v1.1-0.8B-Instant",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.29680998613037446,
     "wilson_low": 0.2779307023854527,
     "wilson_high": 0.316409742250648,
     "right": 642,
     "n": 2163,
     "decision_score": 54.401104026380864,
     "chosen_answer_brier": 0.20878790560275728,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.582727075358276,
     "non_hallucination": 0.11266709102482496,
     "cost_per_1000": 0.0015305129527841948,
     "cost_usd": 0.003310499516872213,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.015555555555555555,
     "median_latency_ms": 23.578784021083266,
     "mixed_concurrency": false
    },
    {
     "name": "Strands Decider 2B",
     "display_name": "Strands Decider 2B",
     "tag": "",
     "class_label": "Open · 2.0B",
     "base_model": null,
     "setup_label": "Strands Decider 2B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.28663892741562647,
     "wilson_low": 0.2679735337100576,
     "wilson_high": 0.30606085809781614,
     "right": 620,
     "n": 2163,
     "decision_score": 53.90008573901988,
     "chosen_answer_brier": 0.20863721263522883,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.5688295737252524,
     "non_hallucination": 0.03170577045022194,
     "cost_per_1000": 0.026221297580520882,
     "cost_usd": 0.056716666666666665,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.02277777777777778,
     "median_latency_ms": 34.699634998105466,
     "mixed_concurrency": false
    },
    {
     "name": "OneJev1.6-0.8B",
     "display_name": "OneJev1.6-0.8B",
     "tag": "",
     "class_label": "Open · 0.8B",
     "base_model": null,
     "setup_label": "OneJev1.6-0.8B",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.2824780397595932,
     "wilson_low": 0.26390351248766303,
     "wilson_high": 0.3018238577086633,
     "right": 611,
     "n": 2163,
     "decision_score": 53.623641032838904,
     "chosen_answer_brier": 0.21000521910281505,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.5921903209205798,
     "non_hallucination": 0.00641025641025641,
     "cost_per_1000": 0.0010932235377029962,
     "cost_usd": 0.002364642512051581,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.011111111111111112,
     "median_latency_ms": 14.550010004313663,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-0.8B-Gen1",
     "display_name": "KnowLine-0.8B-Gen1",
     "tag": "",
     "class_label": "Open · 0.8B",
     "base_model": "Qwen3.5-0.8B",
     "setup_label": "KnowLine-0.8B-Gen1",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.2820157189089228,
     "wilson_low": 0.2634514104152349,
     "wilson_high": 0.3013529573798084,
     "right": 610,
     "n": 2163,
     "decision_score": 53.60218711497086,
     "chosen_answer_brier": 0.20997197660950562,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.5957132150359431,
     "non_hallucination": 0.003223726627981947,
     "cost_per_1000": 0.0016944964834396444,
     "cost_usd": 0.003665195893679951,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.017222222222222222,
     "median_latency_ms": 25.254785985453054,
     "mixed_concurrency": false
    },
    {
     "name": "GLiClass v4 Edge",
     "display_name": "GLiClass v4 Edge",
     "tag": "",
     "class_label": "Open · 0.4B",
     "base_model": "DeBERTa-v3-large",
     "setup_label": "GLiClass v4 Edge",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.22653721682847897,
     "wilson_low": 0.20939025710648737,
     "wilson_high": 0.24465382253459025,
     "right": 490,
     "n": 2163,
     "decision_score": 52.010943991827425,
     "chosen_answer_brier": 0.18631833699193054,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.5011795991558607,
     "non_hallucination": 0.17218171620863712,
     "cost_per_1000": 0.0008472482417198222,
     "cost_usd": 0.0018325979468399754,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.008611111111111111,
     "median_latency_ms": 11.652412998955697,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE 0.4B",
     "display_name": "GLiDE 0.4B",
     "tag": "",
     "class_label": "Open · 0.4B",
     "base_model": null,
     "setup_label": "GLiDE 0.4B (open)",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.20573277854831254,
     "wilson_low": 0.18922578676239032,
     "wilson_high": 0.22328318482095022,
     "right": 445,
     "n": 2163,
     "decision_score": 50.97557265785908,
     "chosen_answer_brier": 0.1862213253911308,
     "n_scored": 2163,
     "decision_score_lacking": 0,
     "auroc": 0.49973970255457745,
     "non_hallucination": 0.7278165503489531,
     "cost_per_1000": 0.0007652564763920974,
     "cost_usd": 0.0016552497584361066,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.0077777777777777776,
     "median_latency_ms": 10.633026016876101,
     "mixed_concurrency": false
    }
   ]
  },
  "together": {
   "name": "Logic and Math",
   "questions": 3288,
   "setups": [
    {
     "name": "GPT-6 Astra",
     "display_name": "GPT-6 Astra",
     "tag": "chat",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "GPT-6 Astra",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9863138686131386,
     "wilson_low": 0.9817369355847908,
     "wilson_high": 0.9897557389108472,
     "right": 3243,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.6086956521739131,
     "cost_per_1000": 0.0,
     "cost_usd": 0.0,
     "cost_unit": "USD",
     "cost_basis": "ChatGPT subscription",
     "gpu_hours": null,
     "median_latency_ms": 5580.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemini 4 Carbon",
     "display_name": "Gemini 4 Carbon",
     "tag": "chat",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "Gemini 4 Carbon (Antigravity subscription)",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9860097323600974,
     "wilson_low": 0.9813903954217115,
     "wilson_high": 0.9894947164257119,
     "right": 3242,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.5333333333333333,
     "cost_per_1000": null,
     "cost_usd": null,
     "cost_unit": "USD",
     "cost_basis": "basis not given",
     "gpu_hours": null,
     "median_latency_ms": 16350.000000000002,
     "mixed_concurrency": false
    },
    {
     "name": "GPT-6.1 Sol",
     "display_name": "GPT-6.1 Sol",
     "tag": "chat",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "GPT-6.1 Sol",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.985705596107056,
     "wilson_low": 0.9810443281008836,
     "wilson_high": 0.9892332210983248,
     "right": 3241,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.6619718309859155,
     "cost_per_1000": 0.0,
     "cost_usd": 0.0,
     "cost_unit": "USD",
     "cost_basis": "ChatGPT subscription",
     "gpu_hours": null,
     "median_latency_ms": 5240.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen4-Max",
     "display_name": "Qwen4-Max",
     "tag": "high · chat",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "Qwen4-Max",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.985705596107056,
     "wilson_low": 0.9810443281008836,
     "wilson_high": 0.9892332210983248,
     "right": 3241,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.10714285714285714,
     "cost_per_1000": 2.9191369525547444,
     "cost_usd": 9.5981223,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 5700.0,
     "mixed_concurrency": false
    },
    {
     "name": "Claude Opus 5.5",
     "display_name": "Claude Opus 5.5",
     "tag": "chat",
     "class_label": "Closed · Flagship",
     "base_model": null,
     "setup_label": "Claude Opus 5.5",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9854014598540146,
     "wilson_low": 0.9806987191203861,
     "wilson_high": 0.9889712674306074,
     "right": 3240,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.4126984126984127,
     "cost_per_1000": 13.179294403892943,
     "cost_usd": 43.33352,
     "cost_unit": "USD",
     "cost_basis": "Claude subscription",
     "gpu_hours": null,
     "median_latency_ms": 3600.0,
     "mixed_concurrency": false
    },
    {
     "name": "DeepSeek-V4.1-Flash",
     "display_name": "DeepSeek-V4.1-Flash",
     "tag": "high · chat · 1017",
     "class_label": "Open · 552B MoE · 16B active",
     "base_model": null,
     "setup_label": "DeepSeek-V4.1-Flash 1017",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9844890510948905,
     "wilson_low": 0.9796645079222253,
     "wilson_high": 0.9881827906841232,
     "right": 3237,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.06349206349206349,
     "cost_per_1000": 0.713104738138686,
     "cost_usd": 2.344688379,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 2270.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemini 4 Hydrogen",
     "display_name": "Gemini 4 Hydrogen",
     "tag": "chat",
     "class_label": "",
     "base_model": null,
     "setup_label": "Gemini 4 Hydrogen (Antigravity subscription)",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9838807785888077,
     "wilson_low": 0.9789770904531233,
     "wilson_high": 0.9876550828567953,
     "right": 3235,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.782608695652174,
     "cost_per_1000": null,
     "cost_usd": null,
     "cost_unit": "USD",
     "cost_basis": "basis not given",
     "gpu_hours": null,
     "median_latency_ms": 4920.0,
     "mixed_concurrency": false
    },
    {
     "name": "Claude Sonnet 5.5",
     "display_name": "Claude Sonnet 5.5",
     "tag": "chat",
     "class_label": "Closed · Workhorse",
     "base_model": null,
     "setup_label": "Claude Sonnet 5.5",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.982360097323601,
     "wilson_low": 0.9772651141529745,
     "wilson_high": 0.9863292459158697,
     "right": 3230,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.43661971830985913,
     "cost_per_1000": 7.042329257907542,
     "cost_usd": 23.1551786,
     "cost_unit": "USD",
     "cost_basis": "Claude subscription",
     "gpu_hours": null,
     "median_latency_ms": 4030.0000000000005,
     "mixed_concurrency": false
    },
    {
     "name": "Inkling 2 Small",
     "display_name": "Inkling 2 Small",
     "tag": "high · chat",
     "class_label": "Open · 594B MoE · 17B active",
     "base_model": null,
     "setup_label": "Inkling 2 Small",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9820559610705596,
     "wilson_low": 0.9769237684351239,
     "wilson_high": 0.9860630289855054,
     "right": 3229,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.4166666666666667,
     "cost_per_1000": 1.8301623600973236,
     "cost_usd": 6.01757384,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 2870.0,
     "mixed_concurrency": false
    },
    {
     "name": "Claude Fable 5.1",
     "display_name": "Claude Fable 5.1",
     "tag": "chat",
     "class_label": "Closed · Flagship",
     "base_model": null,
     "setup_label": "Claude Fable 5.1",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9817518248175182,
     "wilson_low": 0.9765827524100509,
     "wilson_high": 0.9857964823623634,
     "right": 3228,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.15714285714285714,
     "cost_per_1000": 0.0,
     "cost_usd": 0.0,
     "cost_unit": "USD",
     "cost_basis": "Claude subscription",
     "gpu_hours": null,
     "median_latency_ms": 4940.0,
     "mixed_concurrency": false
    },
    {
     "name": "Inkling-Small",
     "display_name": "Inkling-Small",
     "tag": "high · chat · Jul 2026",
     "class_label": "Open · 276B MoE · 12B active",
     "base_model": null,
     "setup_label": "Inkling Small",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9817518248175182,
     "wilson_low": 0.9765827524100509,
     "wilson_high": 0.9857964823623634,
     "right": 3228,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.5060240963855421,
     "cost_per_1000": 2.5056724148418494,
     "cost_usd": 8.2386509,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 2750.0,
     "mixed_concurrency": false
    },
    {
     "name": "Muse Ember",
     "display_name": "Muse Ember",
     "tag": "max · chat",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Muse Ember",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9817518248175182,
     "wilson_low": 0.9765827524100509,
     "wilson_high": 0.9857964823623634,
     "right": 3228,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.31343283582089554,
     "cost_per_1000": 0.8627113211678832,
     "cost_usd": 2.8365948239999996,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 1460.0,
     "mixed_concurrency": false
    },
    {
     "name": "MiMo-V2.6-Flash",
     "display_name": "MiMo-V2.6-Flash",
     "tag": "on · chat",
     "class_label": "Open · 309B MoE · 15B active",
     "base_model": null,
     "setup_label": "MiMo V2.6 Flash",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9796228710462287,
     "wilson_low": 0.9742042433704019,
     "wilson_high": 0.9839220528645081,
     "right": 3221,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.27586206896551724,
     "cost_per_1000": 0.6150036905109489,
     "cost_usd": 2.0221321344,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 10590.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen3.8 Flash",
     "display_name": "Qwen3.8 Flash",
     "tag": "off · chat",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "Qwen3.8 Flash",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9759732360097324,
     "wilson_low": 0.9701571420295234,
     "wilson_high": 0.9806784024268078,
     "right": 3209,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.31313131313131315,
     "cost_per_1000": 0.7320074829683698,
     "cost_usd": 2.406840604,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 6780.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen3.8-27B",
     "display_name": "Qwen3.8-27B",
     "tag": "low · chat",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Qwen3.8-27B (chat, reasoning low, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9744525547445255,
     "wilson_low": 0.9684804774895704,
     "wilson_high": 0.9793172537256861,
     "right": 3204,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.3391304347826087,
     "cost_per_1000": 2.019675081103001,
     "cost_usd": 6.640691666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 2.6669444444444443,
     "median_latency_ms": 11200.0,
     "mixed_concurrency": false
    },
    {
     "name": "GPT-6.1 Terra",
     "display_name": "GPT-6.1 Terra",
     "tag": "chat",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "GPT-6.1 Terra",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9741484184914841,
     "wilson_low": 0.9681457519560306,
     "wilson_high": 0.979044416611011,
     "right": 3203,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.6495726495726496,
     "cost_per_1000": 0.0,
     "cost_usd": 0.0,
     "cost_unit": "USD",
     "cost_basis": "ChatGPT subscription",
     "gpu_hours": null,
     "median_latency_ms": 4440.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 31B",
     "display_name": "Gemma 4 31B",
     "tag": "chat",
     "class_label": "Open · 31B",
     "base_model": null,
     "setup_label": "Gemma 4 31B (chat, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9732360097323601,
     "wilson_low": 0.9671427288495058,
     "wilson_high": 0.9782247517728913,
     "right": 3200,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.16964285714285715,
     "cost_per_1000": 0.7747592254663422,
     "cost_usd": 2.5474083333333333,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 1.0230555555555556,
     "median_latency_ms": 12575.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen3.5-4B",
     "display_name": "Qwen3.5-4B",
     "tag": "on · chat",
     "class_label": "Open · 4.7B",
     "base_model": null,
     "setup_label": "Qwen3.5-4B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9701946472019465,
     "wilson_low": 0.9638109113322746,
     "wilson_high": 0.9754809428079734,
     "right": 3190,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.34234234234234234,
     "cost_per_1000": 2.0421836982968373,
     "cost_usd": 6.714700000000001,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 2.6966666666666668,
     "median_latency_ms": 63270.0,
     "mixed_concurrency": false
    },
    {
     "name": "GLM-5.3-Flash",
     "display_name": "GLM-5.3-Flash",
     "tag": "high · chat",
     "class_label": "Open · 320B MoE · 18B active",
     "base_model": null,
     "setup_label": "GLM-5.3-Flash",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9686739659367397,
     "wilson_low": 0.9621511002339624,
     "wilson_high": 0.9741029406652111,
     "right": 3185,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.07446808510638298,
     "cost_per_1000": 0.4710463503649635,
     "cost_usd": 1.5488004000000002,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 4640.0,
     "mixed_concurrency": false
    },
    {
     "name": "MiniMax-M3.1-Flash",
     "display_name": "MiniMax-M3.1-Flash",
     "tag": "high · chat",
     "class_label": "Open · 428B MoE · 23B active",
     "base_model": null,
     "setup_label": "MiniMax-M3.1-Flash",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.9677615571776156,
     "wilson_low": 0.9611569998080333,
     "wilson_high": 0.9732743531464955,
     "right": 3182,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.3888888888888889,
     "cost_per_1000": 0.34248864507299265,
     "cost_usd": 1.126102665,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 2170.0,
     "mixed_concurrency": false
    },
    {
     "name": "Granite-4.2-3B",
     "display_name": "Granite-4.2-3B",
     "tag": "high · chat",
     "class_label": "Open · 3.0B",
     "base_model": "Granite-4.1-3B-Base",
     "setup_label": "Granite-4.2-3B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9662408759124088,
     "wilson_low": 0.9595029704506822,
     "wilson_high": 0.9718905692627721,
     "right": 3177,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.5466666666666666,
     "cost_per_1000": 4.240665551500406,
     "cost_usd": 13.943308333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 5.599722222222223,
     "median_latency_ms": 25615.000000000004,
     "mixed_concurrency": false
    },
    {
     "name": "MiniCPM5-2B",
     "display_name": "MiniCPM5-2B",
     "tag": "on · chat",
     "class_label": "Open · 2.5B",
     "base_model": null,
     "setup_label": "MiniCPM5-2B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9568126520681265,
     "wilson_low": null,
     "wilson_high": null,
     "right": null,
     "n": null,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.5181347150259067,
     "cost_per_1000": 1.4096985671803193,
     "cost_usd": 4.726158333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 1.8980555555555556,
     "median_latency_ms": 19260.0,
     "mixed_concurrency": false,
     "seed_runs": [
      {
       "seed": 0,
       "run_id": "minicpm5-2b_en+minicpm5-2b_en",
       "right": 3145,
       "n": 3288
      },
      {
       "seed": 1,
       "run_id": "minicpm5-2b-seed1_en+minicpm5-2b-seed1_en",
       "right": 3138,
       "n": 3288
      },
      {
       "seed": 2,
       "run_id": "minicpm5-2b-seed2_en+minicpm5-2b-seed2_en",
       "right": 3155,
       "n": 3288
      }
     ],
     "seeds_complete": "3/3",
     "accuracy_basis": "mean of 3 seeds",
     "seed_min": 0.9543795620437956,
     "seed_max": 0.9595498783454988
    },
    {
     "name": "Qwen3.5-4B",
     "display_name": "Qwen3.5-4B",
     "tag": "off · chat",
     "class_label": "Open · 4.7B",
     "base_model": null,
     "setup_label": "Qwen3.5-4B (chat, thinking off, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9562043795620438,
     "wilson_low": 0.9486609359343924,
     "wilson_high": 0.9626830363879698,
     "right": 3144,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.3395061728395062,
     "cost_per_1000": 0.6832522303325225,
     "cost_usd": 2.2465333333333337,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.9022222222222223,
     "median_latency_ms": 14295.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 12B Unified",
     "display_name": "Gemma 4 12B Unified",
     "tag": "chat",
     "class_label": "Open · 12B",
     "base_model": null,
     "setup_label": "Gemma 4 12B (chat, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9504257907542579,
     "wilson_low": 0.9424663205502152,
     "wilson_high": 0.9573339614560638,
     "right": 3125,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.15337423312883436,
     "cost_per_1000": 0.7493055555555557,
     "cost_usd": 2.463716666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.9894444444444443,
     "median_latency_ms": 22345.0,
     "mixed_concurrency": false
    },
    {
     "name": "LFM2.5-24B-A2B",
     "display_name": "LFM2.5-24B-A2B",
     "tag": "on · chat",
     "class_label": "Open · 24B MoE · 2.5B active",
     "base_model": null,
     "setup_label": "LFM2.5-24B-A2B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.947992700729927,
     "wilson_low": 0.9398666209007278,
     "wilson_high": 0.9550731599198319,
     "right": 3117,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.4397905759162304,
     "cost_per_1000": 3.205058799675588,
     "cost_usd": 10.538233333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 4.232222222222222,
     "median_latency_ms": 42425.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 E4B",
     "display_name": "Gemma 4 E4B",
     "tag": "on · chat · 32k cap · sampled",
     "class_label": "Open · 8.0B",
     "base_model": null,
     "setup_label": "Gemma 4 E4B (chat, thinking on, 32k, sampled, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9421127331711273,
     "wilson_low": null,
     "wilson_high": null,
     "right": null,
     "n": null,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.5411764705882353,
     "cost_per_1000": 0.548551128683428,
     "cost_usd": 1.7948750000000002,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.7208333333333333,
     "median_latency_ms": 13945.0,
     "mixed_concurrency": false,
     "seed_runs": [
      {
       "seed": 0,
       "run_id": "gemma-4-e4b-it-think-32k-sampled-seed0_en+gemma-4-e4b-it-think-32k-sampled-seed0_en",
       "right": 3104,
       "n": 3288
      },
      {
       "seed": 1,
       "run_id": "gemma-4-e4b-it-think-32k-sampled-seed1_en+gemma-4-e4b-it-think-32k-sampled-seed1_en",
       "right": 3099,
       "n": 3288
      },
      {
       "seed": 2,
       "run_id": "gemma-4-e4b-it-think-32k-sampled-seed2_en+gemma-4-e4b-it-think-32k-sampled-seed2_en",
       "right": 3090,
       "n": 3288
      }
     ],
     "seeds_complete": "3/3",
     "accuracy_basis": "mean of 3 seeds",
     "seed_min": 0.9397810218978102,
     "seed_max": 0.9440389294403893
    },
    {
     "name": "LFM3-1.7B",
     "display_name": "LFM3-1.7B",
     "tag": "off · chat",
     "class_label": "Open · 1.7B",
     "base_model": null,
     "setup_label": "LFM3-1.7B (chat, thinking off, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.9419099756690997,
     "wilson_low": 0.9333868061937021,
     "wilson_high": 0.9494017216625594,
     "right": 3097,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.5587044534412956,
     "cost_per_1000": 0.20597129423709634,
     "cost_usd": 0.6772336154515728,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 3.1822222222222223,
     "median_latency_ms": 29040.0,
     "mixed_concurrency": false
    },
    {
     "name": "LFM3-1.7B",
     "display_name": "LFM3-1.7B",
     "tag": "off · chat · NVIDIA RTX PRO 6000 Blackwell",
     "class_label": "Open · 1.7B",
     "base_model": null,
     "setup_label": "LFM3-1.7B (chat, thinking off, RTX PRO 6000, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9401865369018654,
     "wilson_low": null,
     "wilson_high": null,
     "right": null,
     "n": null,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": null,
     "cost_per_1000": 0.44898029872938644,
     "cost_usd": 1.4988416666666668,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.6019444444444444,
     "median_latency_ms": 5666.666666666667,
     "mixed_concurrency": false,
     "seed_runs": [
      {
       "seed": 0,
       "run_id": "lfm3-1.7b-vllm-nothink-rtx-seed0_en+lfm3-1.7b-vllm-nothink-rtx-seed0_en",
       "right": 3098,
       "n": 3288
      },
      {
       "seed": 1,
       "run_id": "lfm3-1.7b-vllm-nothink-rtx-seed1_en+lfm3-1.7b-vllm-nothink-rtx-seed1_en",
       "right": 3085,
       "n": 3288
      },
      {
       "seed": 2,
       "run_id": "lfm3-1.7b-vllm-nothink-rtx-seed2_en+lfm3-1.7b-vllm-nothink-rtx-seed2_en",
       "right": 3091,
       "n": 3288
      }
     ],
     "seeds_complete": "3/3",
     "accuracy_basis": "mean of 3 seeds",
     "seed_min": 0.9382603406326034,
     "seed_max": 0.9422141119221411
    },
    {
     "name": "LFM3-1.7B",
     "display_name": "LFM3-1.7B",
     "tag": "on · chat",
     "class_label": "Open · 1.7B",
     "base_model": null,
     "setup_label": "LFM3-1.7B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9390713706407136,
     "wilson_low": null,
     "wilson_high": null,
     "right": null,
     "n": null,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.4674329501915709,
     "cost_per_1000": 0.43516659908083266,
     "cost_usd": 1.4082333333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.5655555555555556,
     "median_latency_ms": 5630.0,
     "mixed_concurrency": false,
     "seed_runs": [
      {
       "seed": 0,
       "run_id": "lfm3-1.7b-vllm_en+lfm3-1.7b-vllm_en",
       "right": 3089,
       "n": 3288
      },
      {
       "seed": 1,
       "run_id": "lfm3-1.7b-vllm-seed1_en+lfm3-1.7b-vllm-seed1_en",
       "right": 3087,
       "n": 3288
      },
      {
       "seed": 2,
       "run_id": "lfm3-1.7b-vllm-seed2_en+lfm3-1.7b-vllm-seed2_en",
       "right": 3087,
       "n": 3288
      }
     ],
     "seeds_complete": "3/3",
     "accuracy_basis": "mean of 3 seeds",
     "seed_min": 0.9388686131386861,
     "seed_max": 0.9394768856447688
    },
    {
     "name": "Gemma 4 E4B",
     "display_name": "Gemma 4 E4B",
     "tag": "on · chat · 32k cap",
     "class_label": "Open · 8.0B",
     "base_model": null,
     "setup_label": "Gemma 4 E4B (chat, thinking on, 32k, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9388686131386861,
     "wilson_low": 0.9301563367121716,
     "wilson_high": 0.9465565646619406,
     "right": 3087,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": null,
     "cost_per_1000": 0.8549067315490675,
     "cost_usd": 2.8109333333333337,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 1.1288888888888888,
     "median_latency_ms": 25530.0,
     "mixed_concurrency": false
    },
    {
     "name": "MiniCPM5-2B",
     "display_name": "MiniCPM5-2B",
     "tag": "off · chat",
     "class_label": "Open · 2.5B",
     "base_model": null,
     "setup_label": "MiniCPM5-2B (chat, thinking off, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9316707218167073,
     "wilson_low": null,
     "wilson_high": null,
     "right": null,
     "n": null,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.41025641025641024,
     "cost_per_1000": 0.5979859421465262,
     "cost_usd": 2.0743083333333336,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.8330555555555554,
     "median_latency_ms": 9065.0,
     "mixed_concurrency": false,
     "seed_runs": [
      {
       "seed": 0,
       "run_id": "minicpm5-2b-nothink_en+minicpm5-2b-nothink_en",
       "right": 3068,
       "n": 3288
      },
      {
       "seed": 1,
       "run_id": "minicpm5-2b-nothink-seed1_en+minicpm5-2b-nothink-seed1_en",
       "right": 3066,
       "n": 3288
      },
      {
       "seed": 2,
       "run_id": "minicpm5-2b-nothink-seed2_en+minicpm5-2b-nothink-seed2_en",
       "right": 3056,
       "n": 3288
      }
     ],
     "seeds_complete": "3/3",
     "accuracy_basis": "mean of 3 seeds",
     "seed_min": 0.9294403892944039,
     "seed_max": 0.9330900243309003
    },
    {
     "name": "Gemma 4 E4B",
     "display_name": "Gemma 4 E4B",
     "tag": "chat",
     "class_label": "Open · 8.0B",
     "base_model": null,
     "setup_label": "Gemma 4 E4B (chat, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9300486618004866,
     "wilson_low": 0.9208189549606706,
     "wilson_high": 0.9382746296152095,
     "right": 3058,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.5099337748344371,
     "cost_per_1000": 0.466580494728305,
     "cost_usd": 1.5341166666666668,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.616111111111111,
     "median_latency_ms": 13150.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 E4B",
     "display_name": "Gemma 4 E4B",
     "tag": "on · chat",
     "class_label": "Open · 8.0B",
     "base_model": null,
     "setup_label": "Gemma 4 E4B (chat, thinking on, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.9288321167883211,
     "wilson_low": 0.9195343328521712,
     "wilson_high": 0.9371290011308491,
     "right": 3054,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.43727598566308246,
     "cost_per_1000": 0.3754265533314254,
     "cost_usd": 1.2344025073537266,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 5.800277777777778,
     "median_latency_ms": 131535.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 E2B",
     "display_name": "Gemma 4 E2B",
     "tag": "on · chat · 32k cap · sampled",
     "class_label": "Open · 5.1B",
     "base_model": null,
     "setup_label": "Gemma 4 E2B (chat, thinking on, 32k, sampled, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9052108678021087,
     "wilson_low": null,
     "wilson_high": null,
     "right": null,
     "n": null,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.3095890410958904,
     "cost_per_1000": 0.3252179643146797,
     "cost_usd": 1.064475,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.4275,
     "median_latency_ms": 8873.333333333334,
     "mixed_concurrency": false,
     "seed_runs": [
      {
       "seed": 0,
       "run_id": "gemma-4-e2b-it-think-32k-sampled-seed0_en+gemma-4-e2b-it-think-32k-sampled-seed0_en",
       "right": 2966,
       "n": 3288
      },
      {
       "seed": 1,
       "run_id": "gemma-4-e2b-it-think-32k-sampled-seed1_en+gemma-4-e2b-it-think-32k-sampled-seed1_en",
       "right": 2989,
       "n": 3288
      },
      {
       "seed": 2,
       "run_id": "gemma-4-e2b-it-think-32k-sampled-seed2_en+gemma-4-e2b-it-think-32k-sampled-seed2_en",
       "right": 2974,
       "n": 3288
      }
     ],
     "seeds_complete": "3/3",
     "accuracy_basis": "mean of 3 seeds",
     "seed_min": 0.9020681265206812,
     "seed_max": 0.9090632603406326
    },
    {
     "name": "Gemma 4 E2B",
     "display_name": "Gemma 4 E2B",
     "tag": "on · chat · 32k cap",
     "class_label": "Open · 5.1B",
     "base_model": null,
     "setup_label": "Gemma 4 E2B (chat, thinking on, 32k, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.9032846715328468,
     "wilson_low": 0.89270597505989,
     "wilson_high": 0.9129220964730782,
     "right": 2970,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": null,
     "cost_per_1000": 0.6775724858069749,
     "cost_usd": 2.2278583333333337,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.8947222222222222,
     "median_latency_ms": 19100.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 E2B",
     "display_name": "Gemma 4 E2B",
     "tag": "on · chat",
     "class_label": "Open · 5.1B",
     "base_model": null,
     "setup_label": "Gemma 4 E2B (chat, thinking on, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.8929440389294404,
     "wilson_low": 0.8819133264680349,
     "wilson_high": 0.9030576150256263,
     "right": 2936,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.24266666666666667,
     "cost_per_1000": 0.2151047978572469,
     "cost_usd": 0.7072645753546278,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 3.3233333333333333,
     "median_latency_ms": 77885.0,
     "mixed_concurrency": false
    },
    {
     "name": "Gemma 4 E2B",
     "display_name": "Gemma 4 E2B",
     "tag": "chat",
     "class_label": "Open · 5.1B",
     "base_model": null,
     "setup_label": "Gemma 4 E2B (chat, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.8859489051094891,
     "wilson_low": 0.8746301424520613,
     "wilson_high": 0.8963668581326572,
     "right": 2913,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.3233082706766917,
     "cost_per_1000": 0.31259630981346315,
     "cost_usd": 1.0278166666666668,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.4127777777777778,
     "median_latency_ms": 9934.999999999998,
     "mixed_concurrency": false
    },
    {
     "name": "Raon-1 nano",
     "display_name": "Raon-1 nano",
     "tag": "on · chat",
     "class_label": "Open · 3.1B",
     "base_model": "Raon-1 mini preview (MoE-to-Dense pruning and distillation)",
     "setup_label": "Raon-1 nano (chat, thinking on, KRAFTON vLLM fork, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.8807785888077859,
     "wilson_low": 0.8692553154281564,
     "wilson_high": 0.8914131201369087,
     "right": 2896,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": null,
     "cost_per_1000": 6.344064274128143,
     "cost_usd": 20.859283333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 8.377222222222223,
     "median_latency_ms": 54985.0,
     "mixed_concurrency": false
    },
    {
     "name": "LFM2.5-2.6B",
     "display_name": "LFM2.5-2.6B",
     "tag": "on · chat",
     "class_label": "Open · 2.7B",
     "base_model": null,
     "setup_label": "LFM2.5-2.6B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.8762165450121655,
     "wilson_low": 0.864518387553962,
     "wilson_high": 0.8870366082878794,
     "right": 2881,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.4059633027522936,
     "cost_per_1000": 1.0457040754257907,
     "cost_usd": 3.438275,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 1.3808333333333334,
     "median_latency_ms": 14035.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen3.5-2B",
     "display_name": "Qwen3.5-2B",
     "tag": "on · chat",
     "class_label": "Open · 2.0B",
     "base_model": null,
     "setup_label": "Qwen3.5-2B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.8637469586374696,
     "wilson_low": 0.8515954430103364,
     "wilson_high": 0.8750494842546938,
     "right": 2840,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.30913978494623656,
     "cost_per_1000": 3.1063995336577457,
     "cost_usd": 10.213841666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 4.101944444444444,
     "median_latency_ms": 121840.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen3-14B",
     "display_name": "Qwen3-14B",
     "tag": "off · chat",
     "class_label": "Open · 15B",
     "base_model": null,
     "setup_label": "Qwen3-14B (chat, reasoning off, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.8537104622871047,
     "wilson_low": 0.8412180881785131,
     "wilson_high": 0.8653772716954252,
     "right": 2807,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.20930232558139536,
     "cost_per_1000": 0.8397607461476074,
     "cost_usd": 2.7611333333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 1.108888888888889,
     "median_latency_ms": 9380.0,
     "mixed_concurrency": false
    },
    {
     "name": "Qwen3.5-2B",
     "display_name": "Qwen3.5-2B",
     "tag": "off · chat",
     "class_label": "Open · 2.0B",
     "base_model": null,
     "setup_label": "Qwen3.5-2B (chat, thinking off, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.8442822384428224,
     "wilson_low": 0.8314874186431422,
     "wilson_high": 0.8562734991361336,
     "right": 2776,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.3431034482758621,
     "cost_per_1000": 0.7880119626926199,
     "cost_usd": 2.590983333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 1.0405555555555555,
     "median_latency_ms": 17805.0,
     "mixed_concurrency": false
    },
    {
     "name": "LFM2.5-8B-A1B",
     "display_name": "LFM2.5-8B-A1B",
     "tag": "on · chat",
     "class_label": "Open · 8.3B MoE · 1.5B active",
     "base_model": null,
     "setup_label": "LFM2.5-8B-A1B (chat, thinking, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.8391119221411192,
     "wilson_low": 0.8261580677410059,
     "wilson_high": 0.8512742850186166,
     "right": 2759,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.5593705293276109,
     "cost_per_1000": 1.9130221005677215,
     "cost_usd": 6.290016666666668,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 2.5261111111111108,
     "median_latency_ms": 20095.0,
     "mixed_concurrency": true
    },
    {
     "name": "GPT-6 Luna",
     "display_name": "GPT-6 Luna",
     "tag": "chat",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "GPT-6 Luna (low)",
     "group": "chat",
     "gpu_or_api": "API",
     "accuracy": 0.8144768856447688,
     "wilson_low": 0.8008255243248242,
     "wilson_high": 0.8273942539293907,
     "right": 2678,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": null,
     "cost_per_1000": 0.0,
     "cost_usd": 0.0,
     "cost_unit": "USD",
     "cost_basis": "ChatGPT subscription",
     "gpu_hours": null,
     "median_latency_ms": 3830.0,
     "mixed_concurrency": false
    },
    {
     "name": "Raon-1 nano",
     "display_name": "Raon-1 nano",
     "tag": "off · chat",
     "class_label": "Open · 3.1B",
     "base_model": "Raon-1 mini preview (MoE-to-Dense pruning and distillation)",
     "setup_label": "Raon-1 nano (chat, thinking off, KRAFTON vLLM fork, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.8117396593673966,
     "wilson_low": 0.7980165018700709,
     "wilson_high": 0.8247352125502099,
     "right": 2669,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.3113673805601318,
     "cost_per_1000": 4.422207015409571,
     "cost_usd": 14.540216666666668,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 5.839444444444444,
     "median_latency_ms": 42585.0,
     "mixed_concurrency": false
    },
    {
     "name": "Erk-14B",
     "display_name": "Erk-14B",
     "tag": "off · chat",
     "class_label": "Open · 15B",
     "base_model": null,
     "setup_label": "Erk-14B (chat, reasoning off, on our GPU)",
     "group": "chat",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.7825425790754258,
     "wilson_low": 0.7681168283669678,
     "wilson_high": 0.7963088718246819,
     "right": 2573,
     "n": 3288,
     "decision_score": null,
     "chosen_answer_brier": null,
     "n_scored": null,
     "decision_score_lacking": null,
     "auroc": null,
     "non_hallucination": 0.32280701754385965,
     "cost_per_1000": 0.30838909164639094,
     "cost_usd": 1.0139833333333335,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.4072222222222222,
     "median_latency_ms": 8740.0,
     "mixed_concurrency": false
    },
    {
     "name": "Karpo Decision 31B",
     "display_name": "Karpo Decision 31B",
     "tag": "",
     "class_label": "Open · 31B",
     "base_model": null,
     "setup_label": "karpo-decision-31B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.7442214111922141,
     "wilson_low": 0.7290290972182863,
     "wilson_high": 0.7588437092982848,
     "right": 2447,
     "n": 3288,
     "decision_score": 81.13494164450583,
     "chosen_answer_brier": 0.12152257830209745,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8818131061014312,
     "non_hallucination": 0.05714285714285714,
     "cost_per_1000": 0.11801246958637471,
     "cost_usd": 0.388025,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.15583333333333332,
     "median_latency_ms": 161.51595449991873,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-31B-Gen2",
     "display_name": "KnowLine-31B-Gen2",
     "tag": "",
     "class_label": "Open · 31B",
     "base_model": "Gemma 4 31B",
     "setup_label": "KnowLine-31B-Gen2",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.7296228710462287,
     "wilson_low": 0.714179556453135,
     "wilson_high": 0.7445302429491205,
     "right": 2399,
     "n": 3288,
     "decision_score": 80.27311578517886,
     "chosen_answer_brier": 0.12416055534265152,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8817805131590731,
     "non_hallucination": 0.0011441647597254005,
     "cost_per_1000": 0.09297952149229523,
     "cost_usd": 0.3057166666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.12277777777777776,
     "median_latency_ms": 116.40891249408014,
     "mixed_concurrency": false
    },
    {
     "name": "REAX 1.3",
     "display_name": "REAX 1.3",
     "tag": "",
     "class_label": "Open · 57B",
     "base_model": null,
     "setup_label": "REAX 1.3",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.7159367396593674,
     "wilson_low": 0.7002769781855241,
     "wilson_high": 0.7310925020470608,
     "right": 2354,
     "n": 3288,
     "decision_score": 79.29701206693233,
     "chosen_answer_brier": 0.12999649832072077,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8778069675926347,
     "non_hallucination": 0.0380952380952381,
     "cost_per_1000": 0.14830444038929444,
     "cost_usd": 0.4876250000000001,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.19583333333333333,
     "median_latency_ms": 200.24179000000686,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-31B-Gen1.5",
     "display_name": "KnowLine-31B-Gen1.5",
     "tag": "",
     "class_label": "Open · 31B",
     "base_model": null,
     "setup_label": "KnowLine 31B Gen1.5",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.7110705596107056,
     "wilson_low": 0.6953380769221541,
     "wilson_high": 0.7263104009389921,
     "right": 2338,
     "n": 3288,
     "decision_score": 78.92038390650556,
     "chosen_answer_brier": 0.13266288148059432,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8702458241411913,
     "non_hallucination": 0.007454739084132056,
     "cost_per_1000": 0.05185241031597293,
     "cost_usd": 0.170490725118919,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.8011111111111111,
     "median_latency_ms": 811.3891435059486,
     "mixed_concurrency": false
    },
    {
     "name": "Copernicus-31B",
     "display_name": "Copernicus-31B",
     "tag": "",
     "class_label": "Open · 31B",
     "base_model": null,
     "setup_label": "Copernicus-31B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.7095498783454988,
     "wilson_low": 0.6937951165006049,
     "wilson_high": 0.7248155481194667,
     "right": 2333,
     "n": 3288,
     "decision_score": 78.75759952744059,
     "chosen_answer_brier": 0.13439788779668696,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8730336196120763,
     "non_hallucination": 0.01818181818181818,
     "cost_per_1000": 0.10202504055150041,
     "cost_usd": 0.33545833333333336,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.13472222222222222,
     "median_latency_ms": 127.69049649432418,
     "mixed_concurrency": false
    },
    {
     "name": "Sieve 31B v3",
     "display_name": "Sieve 31B v3",
     "tag": "",
     "class_label": "Open · 31B",
     "base_model": null,
     "setup_label": "Sieve 31B v3",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.7052919708029197,
     "wilson_low": 0.689475945522087,
     "wilson_high": 0.720628842022976,
     "right": 2319,
     "n": 3288,
     "decision_score": 78.49255577083333,
     "chosen_answer_brier": 0.13544085538625303,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8649699547552391,
     "non_hallucination": 0.03842159916926272,
     "cost_per_1000": 0.047285658505897644,
     "cost_usd": 0.15547524516739145,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.7305555555555556,
     "median_latency_ms": 675.9455630090088,
     "mixed_concurrency": false
    },
    {
     "name": "Logits Sigma-1 Duo",
     "display_name": "Logits Sigma-1 Duo",
     "tag": "",
     "class_label": "Open · 57B",
     "base_model": null,
     "setup_label": "Logits Sigma-1 Duo",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.6925182481751825,
     "wilson_low": 0.6765281418339402,
     "wilson_high": 0.7080590144860968,
     "right": 2277,
     "n": 3288,
     "decision_score": 77.59943202904066,
     "chosen_answer_brier": 0.14052960759436942,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8679318884453706,
     "non_hallucination": 0.04420432220039293,
     "cost_per_1000": 0.16492295214922953,
     "cost_usd": 0.5422666666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.2177777777777778,
     "median_latency_ms": 207.49457550118677,
     "mixed_concurrency": false
    },
    {
     "name": "d1-ultra",
     "display_name": "d1-ultra",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Liquid d1-ultra",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.6754866180048662,
     "wilson_low": 0.6592863584429558,
     "wilson_high": 0.6912772895770463,
     "right": 2221,
     "n": 3288,
     "decision_score": 76.50126763299767,
     "chosen_answer_brier": 0.145461265344913,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8519676918837694,
     "non_hallucination": 0.8905275099262621,
     "cost_per_1000": 0.008365766423357663,
     "cost_usd": 0.02750664,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 180.41042401455343,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE 27B",
     "display_name": "GLiDE 27B",
     "tag": "",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "GLiDE 27B (open)",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.6733576642335767,
     "wilson_low": 0.657132851448653,
     "wilson_high": 0.6891778580338448,
     "right": 2214,
     "n": 3288,
     "decision_score": 76.29261981431505,
     "chosen_answer_brier": 0.14750526794727564,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.852341793126187,
     "non_hallucination": 0.3189509306260575,
     "cost_per_1000": 0.028551188481888007,
     "cost_usd": 0.09387630772844777,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.4411111111111111,
     "median_latency_ms": 450.0299205101328,
     "mixed_concurrency": false
    },
    {
     "name": "Mammoth-1",
     "display_name": "Mammoth-1",
     "tag": "",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Mammoth-1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.6730535279805353,
     "wilson_low": 0.6568252382268559,
     "wilson_high": 0.6888779086074269,
     "right": 2213,
     "n": 3288,
     "decision_score": 76.68385191187448,
     "chosen_answer_brier": 0.13937648974304562,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8676158849924862,
     "non_hallucination": 0.7136782423812899,
     "cost_per_1000": 0.027978375912408766,
     "cost_usd": 0.09199290000000002,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.15833333333333333,
     "median_latency_ms": 150.05812100002913,
     "mixed_concurrency": false
    },
    {
     "name": "Perplexity Decider 27B v1.3",
     "display_name": "Perplexity Decider 27B v1.3",
     "tag": "NVIDIA RTX PRO 6000 Blackwell",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Perplexity Decider 27B v1.3 (local GPU)",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.6642335766423357,
     "wilson_low": 0.6479077573460107,
     "wilson_high": 0.6801760726900395,
     "right": 2184,
     "n": 3288,
     "decision_score": 75.43979407958633,
     "chosen_answer_brier": 0.15543769505060892,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8438370958751393,
     "non_hallucination": 0.1352253756260434,
     "cost_per_1000": 0.07110198702351987,
     "cost_usd": 0.23378333333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.09388888888888888,
     "median_latency_ms": 96.86675999910221,
     "mixed_concurrency": false
    },
    {
     "name": "d3-ultra",
     "display_name": "d3-ultra",
     "tag": "",
     "class_label": "Open · 26B",
     "base_model": null,
     "setup_label": "vLLM-SR d3-ultra",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.6572384428223844,
     "wilson_low": 0.6408397580149927,
     "wilson_high": 0.6732701311121148,
     "right": 2161,
     "n": 3288,
     "decision_score": 74.3086955172102,
     "chosen_answer_brier": 0.17106453247818046,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8349686936320109,
     "non_hallucination": 0.075,
     "cost_per_1000": 0.09571421330089215,
     "cost_usd": 0.31470833333333337,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.12638888888888888,
     "median_latency_ms": 129.16593151317102,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE",
     "display_name": "GLiDE",
     "tag": "API",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Fastino GLiDE",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.6520681265206812,
     "wilson_low": 0.6356180965431474,
     "wilson_high": 0.6681632275643067,
     "right": 2144,
     "n": 3288,
     "decision_score": 75.03750341862937,
     "chosen_answer_brier": 0.15131805814809385,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8520859970253627,
     "non_hallucination": 0.49231894659839065,
     "cost_per_1000": 0.03641756386861313,
     "cost_usd": 0.11974094999999998,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 266.43565099220723,
     "mixed_concurrency": false
    },
    {
     "name": "d3",
     "display_name": "d3",
     "tag": "",
     "class_label": "Open · 26B",
     "base_model": null,
     "setup_label": "vLLM-SR d3",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.6496350364963503,
     "wilson_low": 0.6331615754896043,
     "wilson_high": 0.6657592474321306,
     "right": 2136,
     "n": 3288,
     "decision_score": 75.1610136388347,
     "chosen_answer_brier": 0.14641476371965625,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8578107931491885,
     "non_hallucination": 0.09333333333333334,
     "cost_per_1000": 0.04627939983779401,
     "cost_usd": 0.15216666666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.06111111111111111,
     "median_latency_ms": 57.1604419999403,
     "mixed_concurrency": false
    },
    {
     "name": "Drex Max v1.1",
     "display_name": "Drex Max v1.1",
     "tag": "",
     "class_label": "Closed · Efficient (provisional)",
     "base_model": null,
     "setup_label": "Drex Max 1.1",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.648418491484185,
     "wilson_low": 0.6319334894675177,
     "wilson_high": 0.6645570828613576,
     "right": 2132,
     "n": 3288,
     "decision_score": 74.60966985596961,
     "chosen_answer_brier": 0.1562250943647928,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8460197468789966,
     "non_hallucination": 0.4826855123674912,
     "cost_per_1000": 0.016127025547445255,
     "cost_usd": 0.053025659999999995,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 327.38201750908047,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE",
     "display_name": "GLiDE",
     "tag": "v1.2",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Fastino GLiDE v1.2",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.648418491484185,
     "wilson_low": 0.6319334894675177,
     "wilson_high": 0.6645570828613576,
     "right": 2132,
     "n": 3288,
     "decision_score": 74.67853270370853,
     "chosen_answer_brier": 0.15484783741001415,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8432109655472386,
     "non_hallucination": 0.4225878833208676,
     "cost_per_1000": 0.009711350364963504,
     "cost_usd": 0.03193092,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.08722222222222223,
     "median_latency_ms": 80.6448784423992,
     "mixed_concurrency": false
    },
    {
     "name": "Drex Max",
     "display_name": "Drex Max",
     "tag": "",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Drex Max v1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.6441605839416058,
     "wilson_low": 0.6276360998369606,
     "wilson_high": 0.6603485954169057,
     "right": 2118,
     "n": 3288,
     "decision_score": 73.19666919575653,
     "chosen_answer_brier": 0.18022720002647527,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8409100263916128,
     "non_hallucination": 0.05738396624472574,
     "cost_per_1000": 0.016307025547445255,
     "cost_usd": 0.0536175,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.08416666666666667,
     "median_latency_ms": 80.84020100068301,
     "mixed_concurrency": false
    },
    {
     "name": "Perplexity Decider 27B v1.2",
     "display_name": "Perplexity Decider 27B v1.2",
     "tag": "API",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Perplexity Decider 27B v1.2 (API)",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.6414233576642335,
     "wilson_low": 0.6248742369994771,
     "wilson_high": 0.657642394420455,
     "right": 2109,
     "n": 3288,
     "decision_score": 74.55948205180381,
     "chosen_answer_brier": 0.15023371662815743,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8493990173379486,
     "non_hallucination": 0.10325203252032521,
     "cost_per_1000": 0.005375675182481752,
     "cost_usd": 0.01767522,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 229.64998850011398,
     "mixed_concurrency": false
    },
    {
     "name": "Gemini Decision 1",
     "display_name": "Gemini Decision 1",
     "tag": "20261111",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Gemini Decision 1",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.6405109489051095,
     "wilson_low": 0.6239537449416507,
     "wilson_high": 0.6567401985336369,
     "right": 2106,
     "n": 3288,
     "decision_score": 74.22998950519364,
     "chosen_answer_brier": 0.1559111588012367,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8384484423683521,
     "non_hallucination": 0.030278232405891982,
     "cost_per_1000": 0.027071380778588807,
     "cost_usd": 0.0890107,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 226.9663190090796,
     "mixed_concurrency": false
    },
    {
     "name": "d1.2-ultra",
     "display_name": "d1.2-ultra",
     "tag": "",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "Liquid d1.2-ultra",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.6359489051094891,
     "wilson_low": 0.6193522464878861,
     "wilson_high": 0.6522282572641779,
     "right": 2091,
     "n": 3288,
     "decision_score": 74.30797568335201,
     "chosen_answer_brier": 0.14978939144244888,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8561144611888402,
     "non_hallucination": 0.9022198731501057,
     "cost_per_1000": 0.008365766423357663,
     "cost_usd": 0.02750664,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 217.05193349953333,
     "mixed_concurrency": false
    },
    {
     "name": "Perplexity Decider 27B v1.2",
     "display_name": "Perplexity Decider 27B v1.2",
     "tag": "API · with web",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Perplexity Decider 27B v1.2 (API, with web)",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.6356447688564477,
     "wilson_low": 0.6190455367250826,
     "wilson_high": 0.6519274043787665,
     "right": 2090,
     "n": 3288,
     "decision_score": 74.2776105995046,
     "chosen_answer_brier": 0.1500925568663557,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8535158278150985,
     "non_hallucination": 0.258498023715415,
     "cost_per_1000": 1.0251138686131387,
     "cost_usd": 3.3705744,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 430.40490900034456,
     "mixed_concurrency": false
    },
    {
     "name": "Turion Jev 31B",
     "display_name": "Turion Jev 31B",
     "tag": "",
     "class_label": "Open · 31B",
     "base_model": null,
     "setup_label": "Turion Jev 31B",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.6329075425790754,
     "wilson_low": 0.616285466938582,
     "wilson_high": 0.6492194103313329,
     "right": 2081,
     "n": 3288,
     "decision_score": 73.88078488351451,
     "chosen_answer_brier": 0.15529184490878528,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8440048380283681,
     "non_hallucination": 0.01888412017167382,
     "cost_per_1000": 0.02001100301029052,
     "cost_usd": 0.06579617789783523,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.30916666666666665,
     "median_latency_ms": 300.74938549660146,
     "mixed_concurrency": false
    },
    {
     "name": "SubQ Snap 1.1",
     "display_name": "SubQ Snap 1.1",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "SubQ Snap 1.1",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.6301703163017032,
     "wilson_low": 0.6135259677948577,
     "wilson_high": 0.646510845641123,
     "right": 2072,
     "n": 3288,
     "decision_score": 72.65840672076321,
     "chosen_answer_brier": 0.17700218188643882,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.82702202613798,
     "non_hallucination": 0.5833333333333334,
     "cost_per_1000": 0.010751350364963504,
     "cost_usd": 0.03535044,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 382.004571496509,
     "mixed_concurrency": false
    },
    {
     "name": "celeris-1-decision",
     "display_name": "celeris-1-decision",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Celeris-1 Decision",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.6222627737226277,
     "wilson_low": 0.60555726300094,
     "wilson_high": 0.6386829215814529,
     "right": 2046,
     "n": 3288,
     "decision_score": 72.82026763990268,
     "chosen_answer_brier": 0.1658574209245742,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8293660069606774,
     "non_hallucination": 0.8717402873869079,
     "cost_per_1000": 0.011254987834549878,
     "cost_usd": 0.0370064,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 387.8826164873317,
     "mixed_concurrency": false
    },
    {
     "name": "celeris-1.1-decision",
     "display_name": "celeris-1.1-decision",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Celeris-1.1 Decision",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.6213503649635036,
     "wilson_low": 0.6046380993443728,
     "wilson_high": 0.6377793972933753,
     "right": 2043,
     "n": 3288,
     "decision_score": 72.78365571776156,
     "chosen_answer_brier": 0.1656772506082725,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8300699616871795,
     "non_hallucination": 0.8671665773968934,
     "cost_per_1000": 0.001125498783454988,
     "cost_usd": 0.00370064,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 381.4161700029217,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-12B-Gen1",
     "display_name": "KnowLine-12B-Gen1",
     "tag": "",
     "class_label": "Open · 12B",
     "base_model": "Gemma 4 12B",
     "setup_label": "KnowLine-12B-Gen1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.6088807785888077,
     "wilson_low": 0.5920823961152056,
     "wilson_high": 0.6254250319457314,
     "right": 2002,
     "n": 3288,
     "decision_score": 71.8631643911406,
     "chosen_answer_brier": 0.1716174907659959,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8100126933719469,
     "non_hallucination": 0.04490106544901065,
     "cost_per_1000": 0.041441098945661,
     "cost_usd": 0.13625833333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.05472222222222223,
     "median_latency_ms": 50.530217005871236,
     "mixed_concurrency": false
    },
    {
     "name": "d1.2",
     "display_name": "d1.2",
     "tag": "",
     "class_label": "Closed · Undisclosed",
     "base_model": null,
     "setup_label": "Liquid d1.2",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.5991484184914841,
     "wilson_low": 0.582290763675077,
     "wilson_high": 0.6157746596429828,
     "right": 1970,
     "n": 3288,
     "decision_score": 71.39533346500122,
     "chosen_answer_brier": 0.17124174919145976,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8123300570777136,
     "non_hallucination": 0.668997668997669,
     "cost_per_1000": 0.003983698296836983,
     "cost_usd": 0.0130984,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 207.8404469993984,
     "mixed_concurrency": false
    },
    {
     "name": "Jev 1.14",
     "display_name": "Jev 1.14",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Jev 1.14.0",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.5933698296836983,
     "wilson_low": 0.5764802353198747,
     "wilson_high": 0.6100414976821021,
     "right": 1951,
     "n": 3288,
     "decision_score": 70.99616332116788,
     "chosen_answer_brier": 0.1734465632603406,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8096762606062442,
     "non_hallucination": 0.1535764375876578,
     "cost_per_1000": 0.021268427007299265,
     "cost_usd": 0.06993058799999999,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 164.02799653587863,
     "mixed_concurrency": false
    },
    {
     "name": "d1.1",
     "display_name": "d1.1",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Liquid d1.1",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.5863746958637469,
     "wilson_low": 0.5694496519113532,
     "wilson_high": 0.6030981401816806,
     "right": 1928,
     "n": 3288,
     "decision_score": 70.69086060720593,
     "chosen_answer_brier": 0.17255748371962837,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8107597784964609,
     "non_hallucination": 0.8602150537634409,
     "cost_per_1000": 0.003983698296836983,
     "cost_usd": 0.0130984,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 173.74163796193898,
     "mixed_concurrency": false
    },
    {
     "name": "Jev 1.13",
     "display_name": "Jev 1.13",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Jev 1.13.0",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.5799878345498783,
     "wilson_low": 0.5630334761699025,
     "wilson_high": 0.5967555003106184,
     "right": 1907,
     "n": 3288,
     "decision_score": 70.41106751824817,
     "chosen_answer_brier": 0.17176648418491483,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8173386133711427,
     "non_hallucination": 0.10208333333333333,
     "cost_per_1000": 0.021268427007299265,
     "cost_usd": 0.06993058799999999,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 166.3421865669079,
     "mixed_concurrency": false
    },
    {
     "name": "Adjudex 9B v5",
     "display_name": "Adjudex 9B v5",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": "Qwen3.5-9B",
     "setup_label": "Adjudex 9B v5",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5672141119221411,
     "wilson_low": 0.5502097898181795,
     "wilson_high": 0.5840615554373152,
     "right": 1865,
     "n": 3288,
     "decision_score": 69.27079073603002,
     "chosen_answer_brier": 0.18179829720154084,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.8016515348195765,
     "non_hallucination": 0.06992532247114731,
     "cost_per_1000": 0.028188361719383617,
     "cost_usd": 0.09268333333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.03722222222222222,
     "median_latency_ms": 38.17214949958725,
     "mixed_concurrency": false
    },
    {
     "name": "Elephant-1",
     "display_name": "Elephant-1",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": null,
     "setup_label": "Elephant-1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5623479318734793,
     "wilson_low": 0.5453275929917014,
     "wilson_high": 0.5792227498923548,
     "right": 1849,
     "n": 3288,
     "decision_score": 68.96178368426696,
     "chosen_answer_brier": 0.1831122581881402,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7974832666907454,
     "non_hallucination": 0.6943115364167995,
     "cost_per_1000": 0.009792431569343065,
     "cost_usd": 0.032197514999999996,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.07694444444444444,
     "median_latency_ms": 79.76101199983532,
     "mixed_concurrency": false
    },
    {
     "name": "Jeffan-9B-1211",
     "display_name": "Jeffan-9B-1211",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": "Qwen3.5-9B",
     "setup_label": "Jeffan-9B-1211",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5583941605839416,
     "wilson_low": 0.5413620265375448,
     "wilson_high": 0.5752900019197177,
     "right": 1836,
     "n": 3288,
     "decision_score": 68.9242179959432,
     "chosen_answer_brier": 0.17990980066507792,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7983348412827023,
     "non_hallucination": 0.13543721236028927,
     "cost_per_1000": 0.032395579886455805,
     "cost_usd": 0.10651666666666668,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.042777777777777776,
     "median_latency_ms": 38.673430502967676,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-9B-Gen2",
     "display_name": "KnowLine-9B-Gen2",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": "Qwen3.5-9B",
     "setup_label": "KnowLine-9B-Gen2",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5456204379562044,
     "wilson_low": 0.5285576243683924,
     "wilson_high": 0.5625767728638439,
     "right": 1794,
     "n": 3288,
     "decision_score": 67.98855247294122,
     "chosen_answer_brier": 0.18584938849738009,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7864807427405647,
     "non_hallucination": 0.10439210693825589,
     "cost_per_1000": 0.03407846715328468,
     "cost_usd": 0.11205000000000001,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.045,
     "median_latency_ms": 42.46198399050627,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE 9B",
     "display_name": "GLiDE 9B",
     "tag": "",
     "class_label": "Open · 7.9B",
     "base_model": null,
     "setup_label": "GLiDE 9B (open)",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.5447080291970803,
     "wilson_low": 0.5276434566821555,
     "wilson_high": 0.561668252605436,
     "right": 1791,
     "n": 3288,
     "decision_score": 67.72454640269551,
     "chosen_answer_brier": 0.19021710114316998,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7856554351957218,
     "non_hallucination": 0.26871980676328505,
     "cost_per_1000": 0.0114528382008581,
     "cost_usd": 0.037656932004421426,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.17694444444444446,
     "median_latency_ms": 177.16426849074196,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-9B-Gen1",
     "display_name": "KnowLine-9B-Gen1",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": "Qwen3.5-9B",
     "setup_label": "KnowLine-9B-Gen1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5447080291970803,
     "wilson_low": 0.5276434566821555,
     "wilson_high": 0.561668252605436,
     "right": 1791,
     "n": 3288,
     "decision_score": 67.9635627346085,
     "chosen_answer_brier": 0.18543677450491017,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7816783762947447,
     "non_hallucination": 0.034143138542350626,
     "cost_per_1000": 0.02987124898621249,
     "cost_usd": 0.09821666666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.03944444444444445,
     "median_latency_ms": 38.494838001497556,
     "mixed_concurrency": false
    },
    {
     "name": "StartLux-Decision-v1.1-9B-Instant",
     "display_name": "StartLux-Decision-v1.1-9B-Instant",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": null,
     "setup_label": "StartLux-Decision v1.1 9B",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.5437956204379562,
     "wilson_low": 0.526729346495143,
     "wilson_high": 0.5607596748478036,
     "right": 1788,
     "n": 3288,
     "decision_score": 67.72580031654994,
     "chosen_answer_brier": 0.18927961410695734,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7791085011185682,
     "non_hallucination": 0.09642401021711366,
     "cost_per_1000": 0.008917751566131265,
     "cost_usd": 0.0293215671494396,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.13777777777777778,
     "median_latency_ms": 142.15879499533912,
     "mixed_concurrency": false
    },
    {
     "name": "Microsoft-Decision-1",
     "display_name": "Microsoft-Decision-1",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": "Qwen3.5-9B",
     "setup_label": "Microsoft-Decision-1",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.5425790754257908,
     "wilson_low": 0.5255106223073214,
     "wilson_high": 0.5595481484427658,
     "right": 1784,
     "n": 3288,
     "decision_score": 67.4217678717497,
     "chosen_answer_brier": 0.1941437179907968,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7690804715079668,
     "non_hallucination": 0.11369346733668342,
     "cost_per_1000": 0.009071080291970804,
     "cost_usd": 0.029825712000000004,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 311.72991247149184,
     "mixed_concurrency": false
    },
    {
     "name": "Dexy-Decision-9B-V2",
     "display_name": "Dexy-Decision-9B-V2",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": "vllm-sr/d3-flash",
     "setup_label": "Dexy-Decision-9B-V2",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.541970802919708,
     "wilson_low": 0.5249012985072347,
     "wilson_high": 0.5589423469464226,
     "right": 1782,
     "n": 3288,
     "decision_score": 66.98366916865061,
     "chosen_answer_brier": 0.20229741954669603,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.794582612311696,
     "non_hallucination": 0.061488673139158574,
     "cost_per_1000": 0.03260594079480941,
     "cost_usd": 0.10720833333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.043055555555555555,
     "median_latency_ms": 45.65966100199148,
     "mixed_concurrency": false
    },
    {
     "name": "Drex 2.0",
     "display_name": "Drex 2.0",
     "tag": "",
     "class_label": "Open · 35B MoE · 3.0B active",
     "base_model": null,
     "setup_label": "Drex 2.0",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.5313260340632603,
     "wilson_low": 0.5142422562870581,
     "wilson_high": 0.5483366964790772,
     "right": 1747,
     "n": 3288,
     "decision_score": 67.47507682451338,
     "chosen_answer_brier": 0.18182449757299268,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7894865286815964,
     "non_hallucination": 0.28137769994162287,
     "cost_per_1000": 0.012167025547445256,
     "cost_usd": 0.04000518,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 338.5253835003823,
     "mixed_concurrency": false
    },
    {
     "name": "vera-medium-omni",
     "display_name": "vera-medium-omni",
     "tag": "",
     "class_label": "Open · 4.2B",
     "base_model": "Qwen3.5-4B",
     "setup_label": "vera-medium-omni",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5246350364963503,
     "wilson_low": 0.5075462705022626,
     "wilson_high": 0.5416663040031449,
     "right": 1725,
     "n": 3288,
     "decision_score": 66.6886413292275,
     "chosen_answer_brier": 0.1908622099118005,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7772700214192328,
     "non_hallucination": 0.07883302296710118,
     "cost_per_1000": 0.024191504460665047,
     "cost_usd": 0.07954166666666668,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.03194444444444444,
     "median_latency_ms": 31.40753149637021,
     "mixed_concurrency": false
    },
    {
     "name": "StartLux-Decision-v1.1-4B-Instant",
     "display_name": "StartLux-Decision-v1.1-4B-Instant",
     "tag": "",
     "class_label": "Open · 4.0B",
     "base_model": null,
     "setup_label": "StartLux-Decision v1.1 4B",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.524330900243309,
     "wilson_low": 0.5072419804187781,
     "wilson_high": 0.5413630314384145,
     "right": 1724,
     "n": 3288,
     "decision_score": 66.32887636586089,
     "chosen_answer_brier": 0.19775337292609108,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7628131286308532,
     "non_hallucination": 0.14387633769322236,
     "cost_per_1000": 0.004279082404716212,
     "cost_usd": 0.014069622946706906,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.0661111111111111,
     "median_latency_ms": 67.69024601089768,
     "mixed_concurrency": false
    },
    {
     "name": "Perplexity Decider Flash v1.3",
     "display_name": "Perplexity Decider Flash v1.3",
     "tag": "",
     "class_label": "Open · 4.0B",
     "base_model": null,
     "setup_label": "Perplexity Decider Flash v1.3",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5209854014598541,
     "wilson_low": 0.5038952074830569,
     "wilson_high": 0.5380266152437718,
     "right": 1713,
     "n": 3288,
     "decision_score": 65.92737547885851,
     "chosen_answer_brier": 0.20243789188268396,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.781581185889417,
     "non_hallucination": 0.1502403846153846,
     "cost_per_1000": 0.02377078264395783,
     "cost_usd": 0.07815833333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.03138888888888889,
     "median_latency_ms": 29.230224499769974,
     "mixed_concurrency": false
    },
    {
     "name": "Cerno 9B v1.2",
     "display_name": "Cerno 9B v1.2",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": null,
     "setup_label": "Cerno 9B v1.2",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.5167274939172749,
     "wilson_low": 0.499636785819473,
     "wilson_high": 0.5337791598323469,
     "right": 1699,
     "n": 3288,
     "decision_score": 64.8420581832421,
     "chosen_answer_brier": 0.2198863302524331,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.776006765168568,
     "non_hallucination": 0.03141025641025641,
     "cost_per_1000": 0.010823561376635125,
     "cost_usd": 0.03558786980637629,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.16722222222222222,
     "median_latency_ms": 163.2868820161093,
     "mixed_concurrency": false
    },
    {
     "name": "Valen-Decision-1.5-9B",
     "display_name": "Valen-Decision-1.5-9B",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": "Qwen3.5-9B",
     "setup_label": "Valen-Decision-1.5-9B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5158150851581509,
     "wilson_low": 0.4987244281671031,
     "wilson_high": 0.5328688295400723,
     "right": 1696,
     "n": 3288,
     "decision_score": 65.99210499035013,
     "chosen_answer_brier": 0.19597298535114827,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7652275973025505,
     "non_hallucination": 0.07099879663056559,
     "cost_per_1000": 0.041441098945661,
     "cost_usd": 0.13625833333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.05472222222222222,
     "median_latency_ms": 58.867783496680204,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine 4B Gen5.5",
     "display_name": "KnowLine 4B Gen5.5",
     "tag": "",
     "class_label": "Open · 4.0B",
     "base_model": null,
     "setup_label": "KnowLine 4B Gen5.5",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.5130778588807786,
     "wilson_low": 0.4959876965548382,
     "wilson_high": 0.530137497318403,
     "right": 1687,
     "n": 3288,
     "decision_score": 65.07389764522897,
     "chosen_answer_brier": 0.21159990597619927,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7520588606631822,
     "non_hallucination": 0.043662825955124315,
     "cost_per_1000": 0.005016235255948837,
     "cost_usd": 0.016493381521559776,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.0775,
     "median_latency_ms": 76.13989399396814,
     "mixed_concurrency": false
    },
    {
     "name": "Perplexity Decider Flash v1.2",
     "display_name": "Perplexity Decider Flash v1.2",
     "tag": "",
     "class_label": "Open · 4.0B",
     "base_model": null,
     "setup_label": "Perplexity Decider Flash v1.2",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5109489051094891,
     "wilson_low": 0.4938594813946692,
     "wilson_high": 0.5280127739410675,
     "right": 1680,
     "n": 3288,
     "decision_score": 65.46626144980283,
     "chosen_answer_brier": 0.20162367611343252,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7688208881189291,
     "non_hallucination": 0.1891891891891892,
     "cost_per_1000": 0.0027178375912408756,
     "cost_usd": 0.00893625,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.04166666666666667,
     "median_latency_ms": 41.16922098910436,
     "mixed_concurrency": false
    },
    {
     "name": "Mercury Decide",
     "display_name": "Mercury Decide",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Inception Mercury Decide",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.5060827250608273,
     "wilson_low": 0.48899615180169725,
     "wilson_high": 0.523155101162601,
     "right": 1664,
     "n": 3288,
     "decision_score": 64.36004161008142,
     "chosen_answer_brier": 0.2188818928591988,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7766412096793293,
     "non_hallucination": 0.1009557945041816,
     "cost_per_1000": 0.0054568795620437965,
     "cost_usd": 0.017942220000000002,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 293.44475601101294,
     "mixed_concurrency": false
    },
    {
     "name": "d1",
     "display_name": "d1",
     "tag": "",
     "class_label": "Closed · Efficient",
     "base_model": null,
     "setup_label": "Liquid d1",
     "group": "decision",
     "gpu_or_api": "API",
     "accuracy": 0.5054744525547445,
     "wilson_low": 0.488388349265488,
     "wilson_high": 0.5225477784023803,
     "right": 1662,
     "n": 3288,
     "decision_score": 65.2860813073512,
     "chosen_answer_brier": 0.19975282640772066,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7567848647800557,
     "non_hallucination": 0.0641025641025641,
     "cost_per_1000": 0.003983698296836983,
     "cost_usd": 0.0130984,
     "cost_unit": "USD",
     "cost_basis": "provider price × reported usage",
     "gpu_hours": null,
     "median_latency_ms": 185.09489996358752,
     "mixed_concurrency": false
    },
    {
     "name": "Open-Jev-27B-v1.1",
     "display_name": "Open-Jev-27B-v1.1",
     "tag": "",
     "class_label": "Open · 27B",
     "base_model": "Qwen3.8-27B + adapter",
     "setup_label": "Open-Jev 27B v1.1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.5045620437956204,
     "wilson_low": 0.48747669281254186,
     "wilson_high": 0.5216367469106817,
     "right": 1659,
     "n": 3288,
     "decision_score": 62.90502713191171,
     "chosen_answer_brier": 0.24646150115738638,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7588050520423414,
     "non_hallucination": 0.14518950437317785,
     "cost_per_1000": 0.21204379562043796,
     "cost_usd": 0.6972,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.28,
     "median_latency_ms": 250.33208950026165,
     "mixed_concurrency": false
    },
    {
     "name": "OneJev-27B",
     "display_name": "OneJev-27B",
     "tag": "",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "OneJev 27B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.4927007299270073,
     "wilson_low": 0.47563032908399766,
     "wilson_high": 0.5097881673588447,
     "right": 1620,
     "n": 3288,
     "decision_score": 65.04996386718607,
     "chosen_answer_brier": 0.1917014525832859,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.769564348521183,
     "non_hallucination": 0.015513126491646777,
     "cost_per_1000": 0.069419099756691,
     "cost_usd": 0.22825,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.09166666666666667,
     "median_latency_ms": 80.3113585002393,
     "mixed_concurrency": false
    },
    {
     "name": "Clef",
     "display_name": "Clef",
     "tag": "",
     "class_label": "Open · 27B",
     "base_model": null,
     "setup_label": "Clef",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.47566909975669097,
     "wilson_low": 0.4586369685615856,
     "wilson_high": 0.49275801958122184,
     "right": 1564,
     "n": 3288,
     "decision_score": 62.92300005352798,
     "chosen_answer_brier": 0.2172090986861314,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7472282757045116,
     "non_hallucination": 0.023976608187134502,
     "cost_per_1000": 0.08146350364963503,
     "cost_usd": 0.267852,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.17777777777777776,
     "median_latency_ms": 185.48455750010362,
     "mixed_concurrency": false
    },
    {
     "name": "Verdin-4B-v1.1",
     "display_name": "Verdin-4B-v1.1",
     "tag": "",
     "class_label": "Open · 4.0B",
     "base_model": "Qwen3.5-4B",
     "setup_label": "Verdin-4B-v1.1",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.4747566909975669,
     "wilson_low": 0.45772716987431417,
     "wilson_high": 0.4918451303238486,
     "right": 1561,
     "n": 3288,
     "decision_score": 63.44410334383976,
     "chosen_answer_brier": 0.2058746241207718,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7469667232598882,
     "non_hallucination": 0.08872767857142858,
     "cost_per_1000": 0.03344738442822385,
     "cost_usd": 0.10997500000000002,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.04416666666666667,
     "median_latency_ms": 46.356985505553894,
     "mixed_concurrency": false
    },
    {
     "name": "Chinese Decision Pointer 4B",
     "display_name": "Chinese Decision Pointer 4B",
     "tag": "",
     "class_label": "Open · 4.0B",
     "base_model": "Qwen3.5-4B",
     "setup_label": "Chinese Decision Pointer 4B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.4698905109489051,
     "wilson_low": 0.4528758737772129,
     "wilson_high": 0.4869754240495113,
     "right": 1545,
     "n": 3288,
     "decision_score": 62.33031737371702,
     "chosen_answer_brier": 0.22328416347456462,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7425920046343488,
     "non_hallucination": 0.0777027027027027,
     "cost_per_1000": 0.025032948094079487,
     "cost_usd": 0.08230833333333334,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.03305555555555555,
     "median_latency_ms": 29.64883549429942,
     "mixed_concurrency": false
    },
    {
     "name": "Clef-Flash",
     "display_name": "Clef-Flash",
     "tag": "",
     "class_label": "Open · 9.0B",
     "base_model": null,
     "setup_label": "Clef-flash",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.4607664233576642,
     "wilson_low": 0.4437840754683314,
     "wilson_high": 0.4778403429119456,
     "right": 1515,
     "n": 3288,
     "decision_score": 62.13263625258516,
     "chosen_answer_brier": 0.21811369830596106,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7067985309529261,
     "non_hallucination": 0.06853070175438597,
     "cost_per_1000": 0.012898388077858881,
     "cost_usd": 0.0424099,
     "cost_unit": "USD",
     "cost_basis": "maker's list price",
     "gpu_hours": 0.09916666666666667,
     "median_latency_ms": 102.76331800014304,
     "mixed_concurrency": false
    },
    {
     "name": "Winnow-12B",
     "display_name": "Winnow-12B",
     "tag": "",
     "class_label": "Open · 12B",
     "base_model": null,
     "setup_label": "Winnow-12B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.4443430656934307,
     "wilson_low": 0.42743330786693057,
     "wilson_high": 0.4613827275097414,
     "right": 1461,
     "n": 3288,
     "decision_score": 61.44074708749485,
     "chosen_answer_brier": 0.21552812394353377,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.737125488948756,
     "non_hallucination": 0.17091836734693877,
     "cost_per_1000": 0.042913625304136255,
     "cost_usd": 0.1411,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.05666666666666667,
     "median_latency_ms": 54.94586449913186,
     "mixed_concurrency": false
    },
    {
     "name": "Cygnet",
     "display_name": "Cygnet",
     "tag": "",
     "class_label": "Open · 12B",
     "base_model": "Gemma 4 12B IT",
     "setup_label": "Cygnet",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.4260948905109489,
     "wilson_low": 0.4092878000772273,
     "wilson_high": 0.44307447640655007,
     "right": 1401,
     "n": 3288,
     "decision_score": 60.875418951279805,
     "chosen_answer_brier": 0.20858651148535276,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.7285497488923613,
     "non_hallucination": 0.07257646448937273,
     "cost_per_1000": 0.02933911192214112,
     "cost_usd": 0.09646700000000001,
     "cost_unit": "USD",
     "cost_basis": "public host's list price (same weights)",
     "gpu_hours": 0.03777777777777778,
     "median_latency_ms": 37.662301000636944,
     "mixed_concurrency": false
    },
    {
     "name": "Metask-Jev-4B",
     "display_name": "Metask-Jev-4B",
     "tag": "",
     "class_label": "Open · 4.5B",
     "base_model": null,
     "setup_label": "Metask-Jev 4B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.36739659367396593,
     "wilson_low": 0.35108151316995084,
     "wilson_high": 0.3840211722083493,
     "right": 1208,
     "n": 3288,
     "decision_score": 57.83162687441334,
     "chosen_answer_brier": 0.21076405618569918,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.6593322163143148,
     "non_hallucination": 0.01971830985915493,
     "cost_per_1000": 0.08582725060827251,
     "cost_usd": 0.2822,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.11333333333333333,
     "median_latency_ms": 117.35533649334684,
     "mixed_concurrency": false
    },
    {
     "name": "StartLux-Decision-v1.1-2B-Instant",
     "display_name": "StartLux-Decision-v1.1-2B-Instant",
     "tag": "",
     "class_label": "Open · 2.0B",
     "base_model": null,
     "setup_label": "StartLux-Decision v1.1 2B",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.3594890510948905,
     "wilson_low": 0.34325980146636315,
     "wilson_high": 0.37604625505834943,
     "right": 1182,
     "n": 3288,
     "decision_score": 57.770077793344655,
     "chosen_answer_brier": 0.2040874952279974,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.6751303181788235,
     "non_hallucination": 0.16411181244364292,
     "cost_per_1000": 0.002355293256377411,
     "cost_usd": 0.007744204226968927,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.03638888888888889,
     "median_latency_ms": 36.47598600946367,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE 2B",
     "display_name": "GLiDE 2B",
     "tag": "",
     "class_label": "Open · 1.9B",
     "base_model": null,
     "setup_label": "GLiDE 2B (open)",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.3521897810218978,
     "wilson_low": 0.33604404292851703,
     "wilson_high": 0.36868051003903773,
     "right": 1158,
     "n": 3288,
     "decision_score": 55.24935426215875,
     "chosen_answer_brier": 0.24720269577872292,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.6373377281536079,
     "non_hallucination": 0.23090507726269316,
     "cost_per_1000": 0.0019417684861737434,
     "cost_usd": 0.0063845347825392685,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.03,
     "median_latency_ms": 28.36126249167137,
     "mixed_concurrency": false
    },
    {
     "name": "d1-2B",
     "display_name": "d1-2B",
     "tag": "",
     "class_label": "Open · 1.9B",
     "base_model": "LFM3-VL-2B",
     "setup_label": "Liquid d1-2B (open weights)",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.3488442822384428,
     "wilson_low": 0.3327382103539468,
     "wilson_high": 0.36530315348324405,
     "right": 1147,
     "n": 3288,
     "decision_score": 56.80795728513858,
     "chosen_answer_brier": 0.21268513653567128,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.6475365543482643,
     "non_hallucination": 0.8247239045244033,
     "cost_per_1000": 0.014304541768045418,
     "cost_usd": 0.04703333333333333,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.01888888888888889,
     "median_latency_ms": 16.697394021321088,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-2B-Gen1",
     "display_name": "KnowLine-2B-Gen1",
     "tag": "",
     "class_label": "Open · 2.0B",
     "base_model": "Qwen3.5-2B",
     "setup_label": "KnowLine-2B-Gen1",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.340632603406326,
     "wilson_low": 0.32462764472595595,
     "wilson_high": 0.3570095276094323,
     "right": 1120,
     "n": 3288,
     "decision_score": 56.03524537452181,
     "chosen_answer_brier": 0.21992769591588993,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.6257833091723775,
     "non_hallucination": 0.06833407771326486,
     "cost_per_1000": 0.003398094850804051,
     "cost_usd": 0.01117293586944372,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.052500000000000005,
     "median_latency_ms": 51.05629551690072,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE 0.8B",
     "display_name": "GLiDE 0.8B",
     "tag": "",
     "class_label": "Open · 0.8B",
     "base_model": null,
     "setup_label": "GLiDE 0.8B (open)",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.3108272506082725,
     "wilson_low": 0.295235443925003,
     "wilson_high": 0.3268605888853242,
     "right": 1022,
     "n": 3288,
     "decision_score": 53.319983335415806,
     "chosen_answer_brier": 0.24442758389995642,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.6018398412333776,
     "non_hallucination": 0.2269357495881384,
     "cost_per_1000": 0.0013844090132905393,
     "cost_usd": 0.004551936835699293,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.021388888888888888,
     "median_latency_ms": 20.403308502864093,
     "mixed_concurrency": false
    },
    {
     "name": "StartLux-Decision-v1.1-0.8B-Instant",
     "display_name": "StartLux-Decision-v1.1-0.8B-Instant",
     "tag": "",
     "class_label": "Open · 0.8B",
     "base_model": "Qwen3.5-0.8B",
     "setup_label": "StartLux-Decision-v1.1-0.8B-Instant",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.2798053527980535,
     "wilson_low": 0.26472496441269927,
     "wilson_high": 0.2953996782797073,
     "right": 920,
     "n": 3288,
     "decision_score": 53.99716628850393,
     "chosen_answer_brier": 0.1998620270279751,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.6050730757931845,
     "non_hallucination": 0.14908350305498982,
     "cost_per_1000": 0.0016181404051447863,
     "cost_usd": 0.005320445652116057,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.025,
     "median_latency_ms": 24.184388006688096,
     "mixed_concurrency": false
    },
    {
     "name": "OneJev1.6-0.8B",
     "display_name": "OneJev1.6-0.8B",
     "tag": "",
     "class_label": "Open · 0.8B",
     "base_model": null,
     "setup_label": "OneJev1.6-0.8B",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.2782846715328467,
     "wilson_low": 0.26323161536281986,
     "wilson_high": 0.2938552140885122,
     "right": 915,
     "n": 3288,
     "decision_score": 53.313515643828524,
     "chosen_answer_brier": 0.21201435865627616,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.5763576575269597,
     "non_hallucination": 0.020117351215423303,
     "cost_per_1000": 0.0011147189457664083,
     "cost_usd": 0.0036651958936799504,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.017222222222222222,
     "median_latency_ms": 14.915587002178654,
     "mixed_concurrency": false
    },
    {
     "name": "KnowLine-0.8B-Gen1",
     "display_name": "KnowLine-0.8B-Gen1",
     "tag": "",
     "class_label": "Open · 0.8B",
     "base_model": "Qwen3.5-0.8B",
     "setup_label": "KnowLine-0.8B-Gen1",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.2767639902676399,
     "wilson_low": 0.2617384853813679,
     "wilson_high": 0.2923105308288897,
     "right": 910,
     "n": 3288,
     "decision_score": 53.55113059794877,
     "chosen_answer_brier": 0.20574137830866446,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.58984510023198,
     "non_hallucination": 0.006286672254819782,
     "cost_per_1000": 0.001797933783494207,
     "cost_usd": 0.005911606280128953,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.027777777777777776,
     "median_latency_ms": 26.194870501058176,
     "mixed_concurrency": false
    },
    {
     "name": "Strands Decider 2B",
     "display_name": "Strands Decider 2B",
     "tag": "",
     "class_label": "Open · 2.0B",
     "base_model": null,
     "setup_label": "Strands Decider 2B",
     "group": "decision",
     "gpu_or_api": "NVIDIA RTX PRO 6000 Blackwell (96 GB)",
     "accuracy": 0.26520681265206814,
     "wilson_low": 0.250397975781411,
     "wilson_high": 0.2805636597966801,
     "right": 872,
     "n": 3288,
     "decision_score": 53.20274124087591,
     "chosen_answer_brier": 0.20115198783454988,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.5732081896682666,
     "non_hallucination": 0.0500406834825061,
     "cost_per_1000": 0.026715835360908356,
     "cost_usd": 0.08784166666666667,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA RTX PRO 6000 Blackwell (96 GB) time while the run was running, first to last answer × $2.49/h",
     "gpu_hours": 0.03527777777777778,
     "median_latency_ms": 35.15580849489197,
     "mixed_concurrency": false
    },
    {
     "name": "GLiClass v4 Edge",
     "display_name": "GLiClass v4 Edge",
     "tag": "",
     "class_label": "Open · 0.4B",
     "base_model": "DeBERTa-v3-large",
     "setup_label": "GLiClass v4 Edge",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.22962287104622872,
     "wilson_low": 0.21556695353962763,
     "wilson_high": 0.2443098521973192,
     "right": 755,
     "n": 3288,
     "decision_score": 52.00235878630077,
     "chosen_answer_brier": 0.18957569532021337,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.510560207904665,
     "non_hallucination": 0.15615615615615616,
     "cost_per_1000": 0.0010068429187567558,
     "cost_usd": 0.003310499516872213,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.015555555555555555,
     "median_latency_ms": 12.109245493775234,
     "mixed_concurrency": false
    },
    {
     "name": "GLiDE 0.4B",
     "display_name": "GLiDE 0.4B",
     "tag": "",
     "class_label": "Open · 0.4B",
     "base_model": null,
     "setup_label": "GLiDE 0.4B (open)",
     "group": "decision",
     "gpu_or_api": "NVIDIA DGX Spark (GB10)",
     "accuracy": 0.21472019464720193,
     "wilson_low": 0.20102150667600258,
     "wilson_high": 0.2290847292984137,
     "right": 706,
     "n": 3288,
     "decision_score": 51.390230002042976,
     "chosen_answer_brier": 0.18691559460634238,
     "n_scored": 3288,
     "decision_score_lacking": 0,
     "auroc": 0.519841548484496,
     "non_hallucination": 0.738973008558262,
     "cost_per_1000": 0.000827049540407335,
     "cost_usd": 0.0027193388888593177,
     "cost_unit": "USD",
     "cost_basis": "estimate: NVIDIA DGX Spark (GB10) time while the run was running, first to last answer × $0.213/h",
     "gpu_hours": 0.012777777777777777,
     "median_latency_ms": 10.989043512381613,
     "mixed_concurrency": false
    }
   ]
  }
 }
}
