{
 "about": "The other benches' bench-scores.json (and a bench's metrics file, as <key>-metrics) as this site's build last read them from their published addresses (sitebuild/all_benches.py), by bench: file null while an address serves none; sha256: of its bytes as read; seen: the first build that read this state.",
 "benches": {
  "logic": {
   "url": "https://zekiworks.github.io/analysis/logic/bench-scores-logic.json",
   "published": true,
   "seen": "2026-11-16",
   "file": {
    "bench": "logic",
    "title": "Logic",
    "url": "https://zekiworks.github.io/analysis/logic/logic/",
    "updated": "2026-11-16T21:03:26Z",
    "score": {
     "zero": "chance: 1 of 5 options right on every question (accuracy 20%)",
     "hundred": "every one of the 1,125 Logic questions right (accuracy 100%)"
    },
    "models": [
     {
      "repo": null,
      "commit": null,
      "model_key": "gpt-6-astra",
      "name": "GPT-6 Astra",
      "display_name": "GPT-6 Astra",
      "tag": "chat",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "GPT-6 Astra",
      "maker": "OpenAI",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 95.88888888888889,
      "raw": {
       "metric": "accuracy",
       "value": 0.9671111111111111,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:gpt-6-astra-sub_en",
       "logic:gpt-6-astra-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "gemini-4-carbon",
      "name": "Gemini 4 Carbon",
      "display_name": "Gemini 4 Carbon",
      "tag": "chat",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "Gemini 4 Carbon (Antigravity subscription)",
      "maker": "Google",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 95.88888888888889,
      "raw": {
       "metric": "accuracy",
       "value": 0.9671111111111111,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:gemini-4-carbon-sub_en",
       "logic:gemini-4-carbon-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "gpt-6.1-sol",
      "name": "GPT-6.1 Sol",
      "display_name": "GPT-6.1 Sol",
      "tag": "chat",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "GPT-6.1 Sol",
      "maker": "OpenAI",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 95.66666666666667,
      "raw": {
       "metric": "accuracy",
       "value": 0.9653333333333334,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:gpt-6.1-sol-sub_en",
       "logic:gpt-6.1-sol-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "qwen4-max",
      "name": "Qwen4-Max",
      "display_name": "Qwen4-Max",
      "tag": "high · chat",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "Qwen4-Max",
      "maker": "Alibaba",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 95.33333333333333,
      "raw": {
       "metric": "accuracy",
       "value": 0.9626666666666667,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:qwen4-max_en",
       "logic:qwen4-max_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "claude-opus-5.5",
      "name": "Claude Opus 5.5",
      "display_name": "Claude Opus 5.5",
      "tag": "chat",
      "class_label": "Closed · Flagship",
      "base_model": null,
      "setup_label": "Claude Opus 5.5",
      "maker": "Anthropic",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 95.22222222222221,
      "raw": {
       "metric": "accuracy",
       "value": 0.9617777777777777,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:claude-opus-5.5-sub_en",
       "logic:claude-opus-5.5-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "deepseek-v4.1-flash-1017",
      "name": "DeepSeek-V4.1-Flash",
      "display_name": "DeepSeek-V4.1-Flash",
      "tag": "high · chat · 1017",
      "class_label": "Open · 552B MoE · 16B active",
      "base_model": null,
      "setup_label": "DeepSeek-V4.1-Flash 1017",
      "maker": "DeepSeek",
      "kind": "chat",
      "params_total": 552.0,
      "params_active": 16.0,
      "quantization": null,
      "score_0_100": 95.22222222222221,
      "raw": {
       "metric": "accuracy",
       "value": 0.9617777777777777,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:deepseek-v4.1-flash-1017_en",
       "logic:deepseek-v4.1-flash-1017_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "gemini-4-hydrogen",
      "name": "Gemini 4 Hydrogen",
      "display_name": "Gemini 4 Hydrogen",
      "tag": "chat",
      "class_label": "",
      "base_model": null,
      "setup_label": "Gemini 4 Hydrogen (Antigravity subscription)",
      "maker": "Google",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 95.11111111111111,
      "raw": {
       "metric": "accuracy",
       "value": 0.9608888888888889,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:gemini-4-hydrogen-sub_en",
       "logic:gemini-4-hydrogen-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "inkling-2-small",
      "name": "Inkling 2 Small",
      "display_name": "Inkling 2 Small",
      "tag": "high · chat",
      "class_label": "Open · 594B MoE · 17B active",
      "base_model": null,
      "setup_label": "Inkling 2 Small",
      "maker": "Thinking Machines",
      "kind": "chat",
      "params_total": 594.0,
      "params_active": 17.0,
      "quantization": null,
      "score_0_100": 94.55555555555556,
      "raw": {
       "metric": "accuracy",
       "value": 0.9564444444444444,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:inkling-2-small_en",
       "logic:inkling-2-small_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "claude-sonnet-5.5",
      "name": "Claude Sonnet 5.5",
      "display_name": "Claude Sonnet 5.5",
      "tag": "chat",
      "class_label": "Closed · Workhorse",
      "base_model": null,
      "setup_label": "Claude Sonnet 5.5",
      "maker": "Anthropic",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 94.33333333333333,
      "raw": {
       "metric": "accuracy",
       "value": 0.9546666666666667,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:claude-sonnet-5.5-sub_en",
       "logic:claude-sonnet-5.5-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "muse-ember",
      "name": "Muse Ember",
      "display_name": "Muse Ember",
      "tag": "max · chat",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Muse Ember",
      "maker": "Meta",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 94.33333333333333,
      "raw": {
       "metric": "accuracy",
       "value": 0.9546666666666667,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:muse-ember_en",
       "logic:muse-ember_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "inkling-small",
      "name": "Inkling-Small",
      "display_name": "Inkling-Small",
      "tag": "high · chat · Jul 2026",
      "class_label": "Open · 276B MoE · 12B active",
      "base_model": null,
      "setup_label": "Inkling Small",
      "maker": "Thinking Machines",
      "kind": "chat",
      "params_total": 276.0,
      "params_active": 12.0,
      "quantization": null,
      "score_0_100": 94.2222222222222,
      "raw": {
       "metric": "accuracy",
       "value": 0.9537777777777777,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:inkling-small-64k_en",
       "logic:inkling-small-64k_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "claude-fable-5.1",
      "name": "Claude Fable 5.1",
      "display_name": "Claude Fable 5.1",
      "tag": "chat",
      "class_label": "Closed · Flagship",
      "base_model": null,
      "setup_label": "Claude Fable 5.1",
      "maker": "Anthropic",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 94.0,
      "raw": {
       "metric": "accuracy",
       "value": 0.952,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:claude-fable-5.1-sub_en",
       "logic:claude-fable-5.1-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "mimo-v2.6-flash",
      "name": "MiMo-V2.6-Flash",
      "display_name": "MiMo-V2.6-Flash",
      "tag": "on · chat",
      "class_label": "Open · 309B MoE · 15B active",
      "base_model": null,
      "setup_label": "MiMo V2.6 Flash",
      "maker": "Xiaomi",
      "kind": "chat",
      "params_total": 309.0,
      "params_active": 15.0,
      "quantization": null,
      "score_0_100": 93.88888888888887,
      "raw": {
       "metric": "accuracy",
       "value": 0.9511111111111111,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:mimo-v2.6-flash-64k_en",
       "logic:mimo-v2.6-flash-64k_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "qwen3.5-397b-a17b",
      "name": "Qwen3.5-397B-A17B",
      "display_name": "Qwen3.5-397B-A17B",
      "tag": "on · chat",
      "class_label": "Open · 397B MoE · 17B active",
      "base_model": null,
      "setup_label": "Qwen3.5-397B-A17B",
      "maker": "Alibaba",
      "kind": "chat",
      "params_total": 397.0,
      "params_active": 17.0,
      "quantization": null,
      "score_0_100": 93.88888888888887,
      "raw": {
       "metric": "accuracy",
       "value": 0.9511111111111111,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:qwen3.5-397b-a17b_en",
       "logic:qwen3.5-397b-a17b_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "Qwen/Qwen3.8-27B",
      "commit": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
      "model_key": "qwen3.8-27b",
      "name": "Qwen3.8-27B",
      "display_name": "Qwen3.8-27B",
      "tag": "low · chat",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Qwen3.8-27B (chat, reasoning low, on our GPU)",
      "maker": "Alibaba",
      "kind": "chat",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 92.55555555555556,
      "raw": {
       "metric": "accuracy",
       "value": 0.9404444444444444,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:qwen3.8-27b-bf16_en_low",
       "logic:qwen3.8-27b-bf16_en_low_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 2.2066666666666666,
      "gpu_cost_usd": 5.4946
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "gpt-6.1-terra",
      "name": "GPT-6.1 Terra",
      "display_name": "GPT-6.1 Terra",
      "tag": "chat",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "GPT-6.1 Terra",
      "maker": "OpenAI",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 92.44444444444444,
      "raw": {
       "metric": "accuracy",
       "value": 0.9395555555555556,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:gpt-6.1-terra-sub_en",
       "logic:gpt-6.1-terra-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "minimax-m3.1-flash",
      "name": "MiniMax-M3.1-Flash",
      "display_name": "MiniMax-M3.1-Flash",
      "tag": "high · chat",
      "class_label": "Open · 428B MoE · 23B active",
      "base_model": null,
      "setup_label": "MiniMax-M3.1-Flash",
      "maker": "MiniMax",
      "kind": "chat",
      "params_total": 428.0,
      "params_active": 23.0,
      "quantization": null,
      "score_0_100": 92.44444444444444,
      "raw": {
       "metric": "accuracy",
       "value": 0.9395555555555556,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:minimax-m3.1-flash_en",
       "logic:minimax-m3.1-flash_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "qwen3.8-flash",
      "name": "Qwen3.8 Flash",
      "display_name": "Qwen3.8 Flash",
      "tag": "off · chat",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "Qwen3.8 Flash",
      "maker": "Alibaba",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 92.44444444444444,
      "raw": {
       "metric": "accuracy",
       "value": 0.9395555555555556,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:qwen3.8-flash_en",
       "logic:qwen3.8-flash_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "google/gemma-4-31B-it",
      "commit": "842da3794eaa0b77d5f08bae87a17459d91ff475",
      "model_key": "gemma-4-31b",
      "name": "Gemma 4 31B",
      "display_name": "Gemma 4 31B",
      "tag": "chat",
      "class_label": "Open · 31B",
      "base_model": null,
      "setup_label": "Gemma 4 31B (chat, on our GPU)",
      "maker": "Google",
      "kind": "chat",
      "params_total": 30.7,
      "params_active": 30.7,
      "quantization": "FP8",
      "score_0_100": 91.66666666666667,
      "raw": {
       "metric": "accuracy",
       "value": 0.9333333333333333,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:gemma-4-31b-it_en_t0",
       "logic:gemma-4-31b-it_en_t0_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); BF16 weights quantized to FP8 by vLLM at load",
      "gpu_hours": 0.5619444444444445,
      "gpu_cost_usd": 1.3992416666666667
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "glm-5.3-flash",
      "name": "GLM-5.3-Flash",
      "display_name": "GLM-5.3-Flash",
      "tag": "high · chat",
      "class_label": "Open · 320B MoE · 18B active",
      "base_model": null,
      "setup_label": "GLM-5.3-Flash",
      "maker": "Z.ai",
      "kind": "chat",
      "params_total": 320.0,
      "params_active": 18.0,
      "quantization": null,
      "score_0_100": 91.44444444444444,
      "raw": {
       "metric": "accuracy",
       "value": 0.9315555555555556,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:glm-5.3-flash_en",
       "logic:glm-5.3-flash_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "Qwen/Qwen3.5-4B",
      "commit": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",
      "model_key": "qwen3.5-4b",
      "name": "Qwen3.5-4B",
      "display_name": "Qwen3.5-4B",
      "tag": "on · chat",
      "class_label": "Open · 4.7B",
      "base_model": null,
      "setup_label": "Qwen3.5-4B (chat, thinking, on our GPU)",
      "maker": "Alibaba",
      "kind": "chat",
      "params_total": 4.66,
      "params_active": 4.66,
      "quantization": "BF16",
      "score_0_100": 90.44444444444444,
      "raw": {
       "metric": "accuracy",
       "value": 0.9235555555555556,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:qwen3.5-4b_en",
       "logic:qwen3.5-4b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: Qwen3.5-4B (off · chat)",
      "gpu_hours": 1.5925,
      "gpu_cost_usd": 3.9653250000000004
     },
     {
      "repo": "ibm-granite/granite-4.2-3b",
      "commit": "4e9c275e232f10211ec88644d106a1f6a1b60e06",
      "model_key": "granite-4.2-3b",
      "name": "Granite-4.2-3B",
      "display_name": "Granite-4.2-3B",
      "tag": "high · chat",
      "class_label": "Open · 3.0B",
      "base_model": "Granite-4.1-3B-Base",
      "setup_label": "Granite-4.2-3B (chat, thinking, on our GPU)",
      "maker": "IBM",
      "kind": "chat",
      "params_total": 3.0,
      "params_active": 3.0,
      "quantization": "BF16",
      "score_0_100": 89.55555555555556,
      "raw": {
       "metric": "accuracy",
       "value": 0.9164444444444444,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:granite-4.2-3b-64k_en",
       "logic:granite-4.2-3b-64k_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 4.921388888888889,
      "gpu_cost_usd": 12.254258333333334
     },
     {
      "repo": "openbmb/MiniCPM5-2B",
      "commit": "42433e90dad95123ab87729fd7011e7110629556",
      "model_key": "minicpm5-2b",
      "name": "MiniCPM5-2B",
      "display_name": "MiniCPM5-2B",
      "tag": "on · chat",
      "class_label": "Open · 2.5B",
      "base_model": null,
      "setup_label": "MiniCPM5-2B (chat, thinking, on our GPU)",
      "maker": "OpenBMB",
      "kind": "chat",
      "params_total": 2.52,
      "params_active": 2.52,
      "quantization": "BF16",
      "score_0_100": 86.07407407407405,
      "raw": {
       "metric": "accuracy",
       "value": 0.8885925925925925,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:minicpm5-2b_en",
       "logic:minicpm5-2b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: MiniCPM5-2B (off · chat); accuracy is the mean of 3 seed runs (min 0.8827, max 0.8960): seed 0 minicpm5-2b_en 998/1125, seed 1 minicpm5-2b-seed1_en 993/1125, seed 2 minicpm5-2b-seed2_en 1008/1125",
      "gpu_hours": 1.5375,
      "gpu_cost_usd": 3.8283750000000003
     },
     {
      "repo": "google/gemma-4-12B-it",
      "commit": "43c9d014b231474fa7212dbb48a762566c2b255d",
      "model_key": "gemma-4-12b",
      "name": "Gemma 4 12B Unified",
      "display_name": "Gemma 4 12B Unified",
      "tag": "chat",
      "class_label": "Open · 12B",
      "base_model": null,
      "setup_label": "Gemma 4 12B (chat, on our GPU)",
      "maker": "Google",
      "kind": "chat",
      "params_total": 11.95,
      "params_active": 11.95,
      "quantization": "BF16",
      "score_0_100": 84.66666666666667,
      "raw": {
       "metric": "accuracy",
       "value": 0.8773333333333333,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:gemma-4-12b-it_en",
       "logic:gemma-4-12b-it_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.5836111111111111,
      "gpu_cost_usd": 1.4531916666666669
     },
     {
      "repo": "LiquidAI/LFM2.5-24B-A2B",
      "commit": "1769516ac588c4c558089c94f27d96b0dc67b183",
      "model_key": "lfm2.5-24b-a2b",
      "name": "LFM2.5-24B-A2B",
      "display_name": "LFM2.5-24B-A2B",
      "tag": "on · chat",
      "class_label": "Open · 24B MoE · 2.5B active",
      "base_model": null,
      "setup_label": "LFM2.5-24B-A2B (chat, thinking, on our GPU)",
      "maker": "Liquid AI",
      "kind": "chat",
      "params_total": 24.0,
      "params_active": 2.45,
      "quantization": "BF16",
      "score_0_100": 83.33333333333334,
      "raw": {
       "metric": "accuracy",
       "value": 0.8666666666666667,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:lfm2.5-24b-a2b_en",
       "logic:lfm2.5-24b-a2b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 3.4163888888888887,
      "gpu_cost_usd": 8.506808333333334
     },
     {
      "repo": "LiquidAI/LFM3-1.7B",
      "commit": "0dc4268b561499c648c7ba7c9466bf3d8d01517a",
      "model_key": "lfm3-1.7b",
      "name": "LFM3-1.7B",
      "display_name": "LFM3-1.7B",
      "tag": "off · chat",
      "class_label": "Open · 1.7B",
      "base_model": null,
      "setup_label": "LFM3-1.7B (chat, thinking off, on our GPU)",
      "maker": null,
      "kind": "chat",
      "params_total": 1.73,
      "params_active": 1.73,
      "quantization": "BF16",
      "score_0_100": 81.8888888888889,
      "raw": {
       "metric": "accuracy",
       "value": 0.8551111111111112,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:lfm3-1.7b-vllm-nothink_en",
       "logic:lfm3-1.7b-vllm-nothink_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10); other setups of this model under its row, not in this entry: LFM3-1.7B (on · chat), LFM3-1.7B (off · chat · NVIDIA RTX PRO 6000 Blackwell)",
      "gpu_hours": 2.5730555555555554,
      "gpu_cost_usd": 0.5475920897283448
     },
     {
      "repo": "google/gemma-4-E4B-it",
      "commit": "4ecc2ba63bcac23ba4a767fc87025b1450f84c9c",
      "model_key": "gemma-4-e4b",
      "name": "Gemma 4 E4B",
      "display_name": "Gemma 4 E4B",
      "tag": "chat",
      "class_label": "Open · 8.0B",
      "base_model": null,
      "setup_label": "Gemma 4 E4B (chat, on our GPU)",
      "maker": "Google",
      "kind": "chat",
      "params_total": 8.0,
      "params_active": 8.0,
      "quantization": "BF16",
      "score_0_100": 78.99999999999999,
      "raw": {
       "metric": "accuracy",
       "value": 0.832,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:gemma-4-e4b-it_en",
       "logic:gemma-4-e4b-it_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: Gemma 4 E4B (on · chat), Gemma 4 E4B (on · chat · 32k cap), Gemma 4 E4B (on · chat · 32k cap · sampled); 4.5B effective parameters",
      "gpu_hours": 0.38055555555555554,
      "gpu_cost_usd": 0.9475833333333333
     },
     {
      "repo": "LiquidAI/LFM2.5-2.6B",
      "commit": "7141b56cd4650af95c7c7b86983443aed050c96a",
      "model_key": "lfm2.5-2.6b",
      "name": "LFM2.5-2.6B",
      "display_name": "LFM2.5-2.6B",
      "tag": "on · chat",
      "class_label": "Open · 2.7B",
      "base_model": null,
      "setup_label": "LFM2.5-2.6B (chat, thinking, on our GPU)",
      "maker": "Liquid AI",
      "kind": "chat",
      "params_total": 2.69,
      "params_active": 2.69,
      "quantization": "BF16",
      "score_0_100": 69.2222222222222,
      "raw": {
       "metric": "accuracy",
       "value": 0.7537777777777778,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:lfm2.5-2.6b_en",
       "logic:lfm2.5-2.6b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 1.1202777777777777,
      "gpu_cost_usd": 2.7894916666666667
     },
     {
      "repo": "google/gemma-4-E2B-it",
      "commit": "4adf15748de9e1a0097b6274cee9c15836c41c37",
      "model_key": "gemma-4-e2b",
      "name": "Gemma 4 E2B",
      "display_name": "Gemma 4 E2B",
      "tag": "chat",
      "class_label": "Open · 5.1B",
      "base_model": null,
      "setup_label": "Gemma 4 E2B (chat, on our GPU)",
      "maker": "Google",
      "kind": "chat",
      "params_total": 5.1,
      "params_active": 5.1,
      "quantization": "BF16",
      "score_0_100": 66.11111111111111,
      "raw": {
       "metric": "accuracy",
       "value": 0.7288888888888889,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:gemma-4-e2b-it_en",
       "logic:gemma-4-e2b-it_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: Gemma 4 E2B (on · chat), Gemma 4 E2B (on · chat · 32k cap), Gemma 4 E2B (on · chat · 32k cap · sampled); 2.3B effective parameters",
      "gpu_hours": 0.23194444444444445,
      "gpu_cost_usd": 0.5775416666666667
     },
     {
      "repo": "Qwen/Qwen3-14B",
      "commit": "40c069824f4251a91eefaf281ebe4c544efd3e18",
      "model_key": "qwen3-14b",
      "name": "Qwen3-14B",
      "display_name": "Qwen3-14B",
      "tag": "off · chat",
      "class_label": "Open · 15B",
      "base_model": null,
      "setup_label": "Qwen3-14B (chat, reasoning off, on our GPU)",
      "maker": "Alibaba",
      "kind": "chat",
      "params_total": 14.8,
      "params_active": 14.8,
      "quantization": "BF16",
      "score_0_100": 56.666666666666664,
      "raw": {
       "metric": "accuracy",
       "value": 0.6533333333333333,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:qwen3-14b_en",
       "logic:qwen3-14b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.8005555555555556,
      "gpu_cost_usd": 1.9933833333333335
     },
     {
      "repo": "KRAFTON/Raon-1-nano",
      "commit": "fcf2d1d55e84e2f9fbe7b5d724dcfc4d861cfcee",
      "model_key": "raon-1-nano",
      "name": "Raon-1 nano",
      "display_name": "Raon-1 nano",
      "tag": "off · chat",
      "class_label": "Open · 3.1B",
      "base_model": "Raon-1 mini preview (MoE-to-Dense pruning and distillation)",
      "setup_label": "Raon-1 nano (chat, thinking off, KRAFTON vLLM fork, on our GPU)",
      "maker": "KRAFTON",
      "kind": "chat",
      "params_total": 3.1,
      "params_active": 3.1,
      "quantization": null,
      "score_0_100": 56.33333333333332,
      "raw": {
       "metric": "accuracy",
       "value": 0.6506666666666666,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:raon-1-nano-nothink_en",
       "logic:raon-1-nano-nothink_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: Raon-1 nano (on · chat); 1.5B effective parameters",
      "gpu_hours": 4.1883333333333335,
      "gpu_cost_usd": 10.42895
     },
     {
      "repo": "Qwen/Qwen3.5-2B",
      "commit": "15852e8c16360a2fea060d615a32b45270f8a8fc",
      "model_key": "qwen3.5-2b",
      "name": "Qwen3.5-2B",
      "display_name": "Qwen3.5-2B",
      "tag": "off · chat",
      "class_label": "Open · 2.0B",
      "base_model": null,
      "setup_label": "Qwen3.5-2B (chat, thinking off, on our GPU)",
      "maker": "Alibaba",
      "kind": "chat",
      "params_total": 2.0,
      "params_active": 2.0,
      "quantization": "BF16",
      "score_0_100": 55.99999999999999,
      "raw": {
       "metric": "accuracy",
       "value": 0.648,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:qwen3.5-2b-nothink_en",
       "logic:qwen3.5-2b-nothink_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: Qwen3.5-2B (on · chat)",
      "gpu_hours": 0.725,
      "gpu_cost_usd": 1.8052500000000002
     },
     {
      "repo": "MachinePulseAI/karpo-decision-31B",
      "commit": "d69cdd57eab23271cbe65236ca8a00d80d7bb8c7",
      "model_key": "karpo-decision-31b",
      "name": "Karpo Decision 31B",
      "display_name": "Karpo Decision 31B",
      "tag": "",
      "class_label": "Open · 31B",
      "base_model": null,
      "setup_label": "karpo-decision-31B",
      "maker": "MachinePulse",
      "kind": "decision",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "BF16",
      "score_0_100": 50.88888888888889,
      "raw": {
       "metric": "accuracy",
       "value": 0.6071111111111112,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.05744902473365083,
      "improvement_points": null,
      "runs": [
       "logic:karpo-decision-31b_en",
       "logic:karpo-decision-31b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.058055555555555555,
      "gpu_cost_usd": 0.14455833333333334
     },
     {
      "repo": "LiquidAI/LFM2.5-8B-A1B",
      "commit": "dd83f5cb02aa2938b66bba9a9998d38d0517c1f3",
      "model_key": "lfm2.5-8b-a1b",
      "name": "LFM2.5-8B-A1B",
      "display_name": "LFM2.5-8B-A1B",
      "tag": "on · chat",
      "class_label": "Open · 8.3B MoE · 1.5B active",
      "base_model": null,
      "setup_label": "LFM2.5-8B-A1B (chat, thinking, on our GPU)",
      "maker": "Liquid AI",
      "kind": "chat",
      "params_total": 8.3,
      "params_active": 1.5,
      "quantization": "BF16",
      "score_0_100": 50.222222222222214,
      "raw": {
       "metric": "accuracy",
       "value": 0.6017777777777777,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:lfm2.5-8b-a1b_en",
       "logic:lfm2.5-8b-a1b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 2.107222222222222,
      "gpu_cost_usd": 5.246983333333334
     },
     {
      "repo": "decisionmodels/REAX-1.3",
      "commit": "fa4bf7ee3bb56a8bc421dad090631be1c5845c2d",
      "model_key": "reax-1.3",
      "name": "REAX 1.3",
      "display_name": "REAX 1.3",
      "tag": "",
      "class_label": "Open · 57B",
      "base_model": null,
      "setup_label": "REAX 1.3",
      "maker": "REAX",
      "kind": "decision",
      "params_total": 57.36,
      "params_active": 57.36,
      "quantization": "FP8",
      "score_0_100": 49.111111111111114,
      "raw": {
       "metric": "accuracy",
       "value": 0.5928888888888889,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.036759860372903616,
      "improvement_points": null,
      "runs": [
       "logic:reax13_en",
       "logic:reax13_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); weights byte-identical to decisionmodels/REAX-1.3 at this commit; served by the REAX engine (github.com/reaxlabs/reax-engine ff8f595)",
      "gpu_hours": 0.0725,
      "gpu_cost_usd": 0.18052500000000002
     },
     {
      "repo": "PelaAI/KnowLine-31B-Gen1.5",
      "commit": "8e56aa1b236a6dd1064dcfd22ba1e5e360e4c140",
      "model_key": "knowline-31b-gen1.5",
      "name": "KnowLine-31B-Gen1.5",
      "display_name": "KnowLine-31B-Gen1.5",
      "tag": "",
      "class_label": "Open · 31B",
      "base_model": null,
      "setup_label": "KnowLine 31B Gen1.5",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "BF16",
      "score_0_100": 48.77777777777777,
      "raw": {
       "metric": "accuracy",
       "value": 0.5902222222222222,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.026296633103252868,
      "improvement_points": null,
      "runs": [
       "logic:knowline-31b_en",
       "logic:knowline-31b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.32472222222222225,
      "gpu_cost_usd": 0.06910667741470745
     },
     {
      "repo": "PelaAI/KnowLine-31B-Gen2",
      "commit": "fd99a9ed2255d8a67dd47a44cc429e638f611593",
      "model_key": "knowline-31b-gen2",
      "name": "KnowLine-31B-Gen2",
      "display_name": "KnowLine-31B-Gen2",
      "tag": "",
      "class_label": "Open · 31B",
      "base_model": "Gemma 4 31B",
      "setup_label": "KnowLine-31B-Gen2",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "BF16",
      "score_0_100": 48.66666666666667,
      "raw": {
       "metric": "accuracy",
       "value": 0.5893333333333334,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.03712864961929971,
      "improvement_points": null,
      "runs": [
       "logic:knowline-31b-gen2_en",
       "logic:knowline-31b-gen2_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.050555555555555555,
      "gpu_cost_usd": 0.12588333333333335
     },
     {
      "repo": "Poindexter-Labs/Copernicus-31B",
      "commit": "be715259e97a743341f2d7a3e6a6298dab23ca0e",
      "model_key": "copernicus-31b",
      "name": "Copernicus-31B",
      "display_name": "Copernicus-31B",
      "tag": "",
      "class_label": "Open · 31B",
      "base_model": null,
      "setup_label": "Copernicus-31B",
      "maker": "Poindexter Labs",
      "kind": "decision",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "BF16",
      "score_0_100": 48.33333333333333,
      "raw": {
       "metric": "accuracy",
       "value": 0.5866666666666667,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.038529006250350314,
      "improvement_points": null,
      "runs": [
       "logic:copernicus-31b_en",
       "logic:copernicus-31b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.06138888888888889,
      "gpu_cost_usd": 0.15285833333333335
     },
     {
      "repo": "fastino/GLiDE-27B",
      "commit": "4d32743ae96b74f0116f050d37742bbd43c04e7d",
      "model_key": "glide-27b",
      "name": "GLiDE 27B",
      "display_name": "GLiDE 27B",
      "tag": "",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "GLiDE 27B (open)",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 47.99999999999999,
      "raw": {
       "metric": "accuracy",
       "value": 0.584,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.04361986569264688,
      "improvement_points": null,
      "runs": [
       "logic:glide-27b_en",
       "logic:glide-27b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.17527777777777778,
      "gpu_cost_usd": 0.037302235627613686
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "gpt-6-luna",
      "name": "GPT-6 Luna",
      "display_name": "GPT-6 Luna",
      "tag": "chat",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "GPT-6 Luna (low)",
      "maker": "OpenAI",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 47.44444444444444,
      "raw": {
       "metric": "accuracy",
       "value": 0.5795555555555556,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:gpt-6-luna-low-sub_en",
       "logic:gpt-6-luna-low-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "sthanika-ai/Sieve-31B-v3",
      "commit": "8fcee858f7f20a50bfa3c8f984c22adb3cbb46a2",
      "model_key": "sieve-31b-v3",
      "name": "Sieve 31B v3",
      "display_name": "Sieve 31B v3",
      "tag": "",
      "class_label": "Open · 31B",
      "base_model": null,
      "setup_label": "Sieve 31B v3",
      "maker": "Sthānika AI",
      "kind": "decision",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "BF16",
      "score_0_100": 47.33333333333333,
      "raw": {
       "metric": "accuracy",
       "value": 0.5786666666666667,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.02063599999999999,
      "improvement_points": null,
      "runs": [
       "logic:sieve-31b_en",
       "logic:sieve-31b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.3327777777777778,
      "gpu_cost_usd": 0.07082104323594485
     },
     {
      "repo": "logitshq/Logits-Sigma-1-Duo",
      "commit": "a0c1b6e8242916aafdcf9a5a49c1a952cb56e091",
      "model_key": "logits-sigma-1-duo",
      "name": "Logits Sigma-1 Duo",
      "display_name": "Logits Sigma-1 Duo",
      "tag": "",
      "class_label": "Open · 57B",
      "base_model": null,
      "setup_label": "Logits Sigma-1 Duo",
      "maker": "Logits",
      "kind": "decision",
      "params_total": 57.36,
      "params_active": 57.36,
      "quantization": "FP8",
      "score_0_100": 47.22222222222221,
      "raw": {
       "metric": "accuracy",
       "value": 0.5777777777777777,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.04976511336326297,
      "improvement_points": null,
      "runs": [
       "logic:logits-sigma-1-duo_en",
       "logic:logits-sigma-1-duo_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.09555555555555556,
      "gpu_cost_usd": 0.23793333333333336
     },
     {
      "repo": "fastino/GLiDE",
      "commit": "a7e45896be7c96ba3354891da790c5ec99c30b6c",
      "model_key": "fastino-glide-v1.2",
      "name": "GLiDE",
      "display_name": "GLiDE",
      "tag": "v1.2",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Fastino GLiDE v1.2",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 44.0,
      "raw": {
       "metric": "accuracy",
       "value": 0.552,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.058800059974995815,
      "improvement_points": null,
      "runs": [
       "logic:glide-v1.2_en",
       "logic:glide-v1.2_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.036944444444444446,
      "gpu_cost_usd": 0.09199166666666667
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "liquid-d1-ultra",
      "name": "d1-ultra",
      "display_name": "d1-ultra",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Liquid d1-ultra",
      "maker": "Liquid AI",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 43.666666666666664,
      "raw": {
       "metric": "accuracy",
       "value": 0.5493333333333333,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.031652433797910245,
      "improvement_points": null,
      "runs": [
       "logic:d1-ultra_en",
       "logic:d1-ultra_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "drex-max-1.1",
      "name": "Drex Max v1.1",
      "display_name": "Drex Max v1.1",
      "tag": "",
      "class_label": "Closed · Efficient (provisional)",
      "base_model": null,
      "setup_label": "Drex Max 1.1",
      "maker": "Nace.AI",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 43.44444444444444,
      "raw": {
       "metric": "accuracy",
       "value": 0.5475555555555556,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.06129836727146942,
      "improvement_points": null,
      "runs": [
       "logic:drex-max-1.1_en",
       "logic:drex-max-1.1_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "perplexity-ai/pplx-decider-v1.3",
      "commit": "9c2ebb27470d3dec2ad7ab727727e40d142c7213",
      "model_key": "perplexity-decider-27b-v1.3",
      "name": "Perplexity Decider 27B v1.3",
      "display_name": "Perplexity Decider 27B v1.3",
      "tag": "NVIDIA RTX PRO 6000 Blackwell",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Perplexity Decider 27B v1.3 (local GPU)",
      "maker": "Perplexity",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 43.44444444444444,
      "raw": {
       "metric": "accuracy",
       "value": 0.5475555555555556,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.09027336492716864,
      "improvement_points": null,
      "runs": [
       "logic:decider-v1.3-local_en",
       "logic:decider-v1.3-local_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.03666666666666667,
      "gpu_cost_usd": 0.0913
     },
     {
      "repo": "vllm-sr/d3-ultra",
      "commit": "52ba9e7a00402f9560c7fba8c039608e7ddee7fb",
      "model_key": "vllm-sr-d3-ultra",
      "name": "d3-ultra",
      "display_name": "d3-ultra",
      "tag": "",
      "class_label": "Open · 26B",
      "base_model": null,
      "setup_label": "vLLM-SR d3-ultra",
      "maker": "vLLM Semantic Router",
      "kind": "decision",
      "params_total": 26.09,
      "params_active": 26.09,
      "quantization": "BF16",
      "score_0_100": 43.22222222222222,
      "raw": {
       "metric": "accuracy",
       "value": 0.5457777777777778,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.15079410528421047,
      "improvement_points": null,
      "runs": [
       "logic:d3-ultra_en",
       "logic:d3-ultra_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.049444444444444444,
      "gpu_cost_usd": 0.12311666666666668
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "fastino-glide",
      "name": "GLiDE",
      "display_name": "GLiDE",
      "tag": "API",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Fastino GLiDE",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 43.11111111111111,
      "raw": {
       "metric": "accuracy",
       "value": 0.5448888888888889,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.06557253261102865,
      "improvement_points": null,
      "runs": [
       "logic:glide_en",
       "logic:glide_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "baselabs/mammoth-1",
      "commit": "61cec44c281c693992e3d6ebeb44dd3402dfa186",
      "model_key": "mammoth-1",
      "name": "Mammoth-1",
      "display_name": "Mammoth-1",
      "tag": "",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Mammoth-1",
      "maker": "Base Labs",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 43.11111111111111,
      "raw": {
       "metric": "accuracy",
       "value": 0.5448888888888889,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.052732048888888904,
      "improvement_points": null,
      "runs": [
       "logic:mammoth-1_en",
       "logic:mammoth-1_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.06805555555555555,
      "gpu_cost_usd": 0.16945833333333335
     },
     {
      "repo": "vllm-sr/d3",
      "commit": "20f37b9a892acc13d4b07a4f3c4f11b2518c615c",
      "model_key": "vllm-sr-d3",
      "name": "d3",
      "display_name": "d3",
      "tag": "",
      "class_label": "Open · 26B",
      "base_model": null,
      "setup_label": "vLLM-SR d3",
      "maker": "vLLM Semantic Router",
      "kind": "decision",
      "params_total": 26.09,
      "params_active": 26.09,
      "quantization": "BF16",
      "score_0_100": 43.11111111111111,
      "raw": {
       "metric": "accuracy",
       "value": 0.5448888888888889,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.046466520101448576,
      "improvement_points": null,
      "runs": [
       "logic:d3_en",
       "logic:d3_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.025833333333333333,
      "gpu_cost_usd": 0.06432500000000001
     },
     {
      "repo": "ecloudtech/Erk-14B",
      "commit": "8584b14a829d5cc803bc227f44e9012e36e8ad04",
      "model_key": "erk-14b",
      "name": "Erk-14B",
      "display_name": "Erk-14B",
      "tag": "off · chat",
      "class_label": "Open · 15B",
      "base_model": null,
      "setup_label": "Erk-14B (chat, reasoning off, on our GPU)",
      "maker": "eCloud Technology",
      "kind": "chat",
      "params_total": 14.77,
      "params_active": 14.77,
      "quantization": "BF16",
      "score_0_100": 42.77777777777778,
      "raw": {
       "metric": "accuracy",
       "value": 0.5422222222222223,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:erk-14b_en",
       "logic:erk-14b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.1761111111111111,
      "gpu_cost_usd": 0.43851666666666667
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "perplexity-decider-27b-v1.2",
      "name": "Perplexity Decider 27B v1.2",
      "display_name": "Perplexity Decider 27B v1.2",
      "tag": "API",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Perplexity Decider 27B v1.2 (API)",
      "maker": "Perplexity",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": null,
      "score_0_100": 41.666666666666664,
      "raw": {
       "metric": "accuracy",
       "value": 0.5333333333333333,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.0340667680314301,
      "improvement_points": null,
      "runs": [
       "logic:decider-v1.2_en",
       "logic:decider-v1.2_en_dontknow"
      ],
      "notes": "hosted API (the maker's service); other setups of this model under its row, not in this entry: Perplexity Decider 27B v1.2 (NVIDIA RTX PRO 6000 Blackwell)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "nace-ai/drex-max-v1",
      "commit": "cac65e1982a0237e996419ba559004522d9a33ac",
      "model_key": "drex-max-v1",
      "name": "Drex Max",
      "display_name": "Drex Max",
      "tag": "",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Drex Max v1",
      "maker": "Nace.AI",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 41.33333333333332,
      "raw": {
       "metric": "accuracy",
       "value": 0.5306666666666666,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.1617667780560019,
      "improvement_points": null,
      "runs": [
       "logic:drex-max-v1_en",
       "logic:drex-max-v1_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); served with Nace's drex-decision-models code at e206b27",
      "gpu_hours": 0.03666666666666667,
      "gpu_cost_usd": 0.0913
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "perplexity-decider-27b-v1.2-with-web",
      "name": "Perplexity Decider 27B v1.2",
      "display_name": "Perplexity Decider 27B v1.2",
      "tag": "API · with web",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Perplexity Decider 27B v1.2 (API, with web)",
      "maker": "Perplexity",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": null,
      "score_0_100": 40.666666666666664,
      "raw": {
       "metric": "accuracy",
       "value": 0.5253333333333333,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.042326970442519755,
      "improvement_points": null,
      "runs": [
       "logic:decider-v1.2-web_en",
       "logic:decider-v1.2-web_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "subq-snap-1.1",
      "name": "SubQ Snap 1.1",
      "display_name": "SubQ Snap 1.1",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "SubQ Snap 1.1",
      "maker": "Subquadratic",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 40.444444444444436,
      "raw": {
       "metric": "accuracy",
       "value": 0.5235555555555556,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.12856464571423,
      "improvement_points": null,
      "runs": [
       "logic:subq-1.1_en",
       "logic:subq-1.1_en_dontknow"
      ],
      "notes": "hosted API (the maker's service); shown as one row with SubQ Snap Preview (the same answer to every question); every number from SubQ Snap 1.1",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "turion-ai/turion-jev-31b",
      "commit": "866c0fc6e040cba23fcc72113a80e5c99d5ebab1",
      "model_key": "turion-jev-31b",
      "name": "Turion Jev 31B",
      "display_name": "Turion Jev 31B",
      "tag": "",
      "class_label": "Open · 31B",
      "base_model": null,
      "setup_label": "Turion Jev 31B",
      "maker": "Turion.AI",
      "kind": "decision",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "FP8",
      "score_0_100": 40.33333333333332,
      "raw": {
       "metric": "accuracy",
       "value": 0.5226666666666666,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.029210571555555558,
      "improvement_points": null,
      "runs": [
       "logic:turion-jev-31b_en",
       "logic:turion-jev-31b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.13166666666666665,
      "gpu_cost_usd": 0.028021013767811232
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "liquid-d1.2-ultra",
      "name": "d1.2-ultra",
      "display_name": "d1.2-ultra",
      "tag": "",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "Liquid d1.2-ultra",
      "maker": "Liquid AI",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 39.444444444444436,
      "raw": {
       "metric": "accuracy",
       "value": 0.5155555555555555,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.030123540871246392,
      "improvement_points": null,
      "runs": [
       "logic:d1.2-ultra_en",
       "logic:d1.2-ultra_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "PelaAI/KnowLine-12B-Gen1",
      "commit": "f26fbc2d64092bb2f8caaedbec243172bb3e908f",
      "model_key": "knowline-12b-gen1",
      "name": "KnowLine-12B-Gen1",
      "display_name": "KnowLine-12B-Gen1",
      "tag": "",
      "class_label": "Open · 12B",
      "base_model": "Gemma 4 12B",
      "setup_label": "KnowLine-12B-Gen1",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 12.0,
      "params_active": 12.0,
      "quantization": "BF16",
      "score_0_100": 38.22222222222222,
      "raw": {
       "metric": "accuracy",
       "value": 0.5057777777777778,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.04789807355469754,
      "improvement_points": null,
      "runs": [
       "logic:knowline-12b-gen1_en",
       "logic:knowline-12b-gen1_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.023333333333333334,
      "gpu_cost_usd": 0.058100000000000006
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "gemini-decision-1",
      "name": "Gemini Decision 1",
      "display_name": "Gemini Decision 1",
      "tag": "20261111",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Gemini Decision 1",
      "maker": "Google",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 37.55555555555555,
      "raw": {
       "metric": "accuracy",
       "value": 0.5004444444444445,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.03088609070996079,
      "improvement_points": null,
      "runs": [
       "logic:gemini-decision-1_en",
       "logic:gemini-decision-1_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "liquid-d1.2",
      "name": "d1.2",
      "display_name": "d1.2",
      "tag": "",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "Liquid d1.2",
      "maker": "Liquid AI",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 36.44444444444444,
      "raw": {
       "metric": "accuracy",
       "value": 0.4915555555555556,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.026105559003673215,
      "improvement_points": null,
      "runs": [
       "logic:d1.2_en",
       "logic:d1.2_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "tattrongvu/Jeffan-9B-1211",
      "commit": "862d1a7a8b64a8d708ddb0b0cd775a740f01841a",
      "model_key": "jeffan-9b-1211",
      "name": "Jeffan-9B-1211",
      "display_name": "Jeffan-9B-1211",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": "Qwen3.5-9B",
      "setup_label": "Jeffan-9B-1211",
      "maker": "tattrongvu",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 35.22222222222222,
      "raw": {
       "metric": "accuracy",
       "value": 0.4817777777777778,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.040072038006967466,
      "improvement_points": null,
      "runs": [
       "logic:jeffan-9b_en",
       "logic:jeffan-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.017777777777777778,
      "gpu_cost_usd": 0.04426666666666667
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "jev-1.14.0",
      "name": "Jev 1.14",
      "display_name": "Jev 1.14",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Jev 1.14.0",
      "maker": "TypeSafe",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 35.22222222222222,
      "raw": {
       "metric": "accuracy",
       "value": 0.4817777777777778,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.03322666666666665,
      "improvement_points": null,
      "runs": [
       "logic:jev-1.14_en",
       "logic:jev-1.14_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "fastino/GLiDE-9B",
      "commit": "ab54df5acba0eebf7df45560ffdaacd64e32df3b",
      "model_key": "glide-9b",
      "name": "GLiDE 9B",
      "display_name": "GLiDE 9B",
      "tag": "",
      "class_label": "Open · 7.9B",
      "base_model": null,
      "setup_label": "GLiDE 9B (open)",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": 7.94,
      "params_active": 7.94,
      "quantization": "BF16",
      "score_0_100": 34.77777777777777,
      "raw": {
       "metric": "accuracy",
       "value": 0.4782222222222222,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.06687502216680195,
      "improvement_points": null,
      "runs": [
       "logic:glide-9b_en",
       "logic:glide-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.07027777777777777,
      "gpu_cost_usd": 0.01495636388872625
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "celeris-1-decision",
      "name": "celeris-1-decision",
      "display_name": "celeris-1-decision",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Celeris-1 Decision",
      "maker": "Celeris",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 34.11111111111111,
      "raw": {
       "metric": "accuracy",
       "value": 0.4728888888888889,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.10650666666666664,
      "improvement_points": null,
      "runs": [
       "logic:celeris_en",
       "logic:celeris_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "celeris-1.1-decision",
      "name": "celeris-1.1-decision",
      "display_name": "celeris-1.1-decision",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Celeris-1.1 Decision",
      "maker": "Celeris",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 34.11111111111111,
      "raw": {
       "metric": "accuracy",
       "value": 0.4728888888888889,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.09840888888888888,
      "improvement_points": null,
      "runs": [
       "logic:celeris-1.1_en",
       "logic:celeris-1.1_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "startlux-models/StartLux-Decision-v1.1-9B-Instant",
      "commit": "18a17f734966b213b2f00fa9c915e8b9af2a9b20",
      "model_key": "startlux-decision-v1.1-9b",
      "name": "StartLux-Decision-v1.1-9B-Instant",
      "display_name": "StartLux-Decision-v1.1-9B-Instant",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": null,
      "setup_label": "StartLux-Decision v1.1 9B",
      "maker": "StartLux",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 33.22222222222222,
      "raw": {
       "metric": "accuracy",
       "value": 0.4657777777777778,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.03781032173777334,
      "improvement_points": null,
      "runs": [
       "logic:startlux-9b_en",
       "logic:startlux-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.05305555555555556,
      "gpu_cost_usd": 0.011291167995046298
     },
     {
      "repo": "newsbang/adjudex-9b-v5",
      "commit": "ffdcb8254d1b22e40b2bbbf569f143e0b357aeca",
      "model_key": "adjudex-9b-v5",
      "name": "Adjudex 9B v5",
      "display_name": "Adjudex 9B v5",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": "Qwen3.5-9B",
      "setup_label": "Adjudex 9B v5",
      "maker": "newsbang",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 33.0,
      "raw": {
       "metric": "accuracy",
       "value": 0.464,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.07197307821973589,
      "improvement_points": null,
      "runs": [
       "logic:adjudex-9b_en",
       "logic:adjudex-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.014166666666666666,
      "gpu_cost_usd": 0.035275
     },
     {
      "repo": "baselabs/elephant-1",
      "commit": "b3f8a93462fb1051a9de031e94a7ff380e254072",
      "model_key": "elephant-1",
      "name": "Elephant-1",
      "display_name": "Elephant-1",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": null,
      "setup_label": "Elephant-1",
      "maker": "Base Labs",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 33.0,
      "raw": {
       "metric": "accuracy",
       "value": 0.464,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.08550222044444446,
      "improvement_points": null,
      "runs": [
       "logic:elephant-1_en",
       "logic:elephant-1_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.02638888888888889,
      "gpu_cost_usd": 0.06570833333333334
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "jev-1.13.0",
      "name": "Jev 1.13",
      "display_name": "Jev 1.13",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Jev 1.13.0",
      "maker": "TypeSafe",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 32.888888888888886,
      "raw": {
       "metric": "accuracy",
       "value": 0.4631111111111111,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.05926222222222222,
      "improvement_points": null,
      "runs": [
       "logic:jev-1.13_en",
       "logic:jev-1.13_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "liquid-d1.1",
      "name": "d1.1",
      "display_name": "d1.1",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Liquid d1.1",
      "maker": "Liquid AI",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 32.55555555555555,
      "raw": {
       "metric": "accuracy",
       "value": 0.46044444444444443,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.04386741270175636,
      "improvement_points": null,
      "runs": [
       "logic:d1.1_en",
       "logic:d1.1_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "startlux-models/StartLux-Decision-v1.1-4B-Instant",
      "commit": "416088f77cfce09633906a3675c7cd2e5dabcf10",
      "model_key": "startlux-decision-v1.1-4b",
      "name": "StartLux-Decision-v1.1-4B-Instant",
      "display_name": "StartLux-Decision-v1.1-4B-Instant",
      "tag": "",
      "class_label": "Open · 4.0B",
      "base_model": null,
      "setup_label": "StartLux-Decision v1.1 4B",
      "maker": "StartLux",
      "kind": "decision",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "BF16",
      "score_0_100": 32.33333333333333,
      "raw": {
       "metric": "accuracy",
       "value": 0.45866666666666667,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.032540590315341435,
      "improvement_points": null,
      "runs": [
       "logic:startlux-4b_en",
       "logic:startlux-4b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.024444444444444446,
      "gpu_cost_usd": 0.005202213526513478
     },
     {
      "repo": "Patrick-18/Dexy-Decision-9B-V2",
      "commit": "4f434e2b5fd9f0addfe3233489571c2dc8ce7a19",
      "model_key": "dexy-decision-9b-v2",
      "name": "Dexy-Decision-9B-V2",
      "display_name": "Dexy-Decision-9B-V2",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": "vllm-sr/d3-flash",
      "setup_label": "Dexy-Decision-9B-V2",
      "maker": "Patrick-18",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 31.66666666666666,
      "raw": {
       "metric": "accuracy",
       "value": 0.4533333333333333,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.14941316223793505,
      "improvement_points": null,
      "runs": [
       "logic:dexy-9b_en",
       "logic:dexy-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.015277777777777777,
      "gpu_cost_usd": 0.03804166666666667
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "inception-mercury-decide",
      "name": "Mercury Decide",
      "display_name": "Mercury Decide",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Inception Mercury Decide",
      "maker": "Inception",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 31.33333333333333,
      "raw": {
       "metric": "accuracy",
       "value": 0.45066666666666666,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.15368720258223856,
      "improvement_points": null,
      "runs": [
       "logic:mercury-decide_en",
       "logic:mercury-decide_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "perplexity-ai/pplx-decider-flash-v1.3",
      "commit": "d06fb3f2d97e0a52325a0d463c450cf0338f44fe",
      "model_key": "perplexity-decider-flash-v1.3",
      "name": "Perplexity Decider Flash v1.3",
      "display_name": "Perplexity Decider Flash v1.3",
      "tag": "",
      "class_label": "Open · 4.0B",
      "base_model": null,
      "setup_label": "Perplexity Decider Flash v1.3",
      "maker": "Perplexity",
      "kind": "decision",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "BF16",
      "score_0_100": 30.88888888888889,
      "raw": {
       "metric": "accuracy",
       "value": 0.4471111111111111,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.10082020218574833,
      "improvement_points": null,
      "runs": [
       "logic:decider-flash-v1.3_en",
       "logic:decider-flash-v1.3_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.011944444444444445,
      "gpu_cost_usd": 0.02974166666666667
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "microsoft-decision-1",
      "name": "Microsoft-Decision-1",
      "display_name": "Microsoft-Decision-1",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": "Qwen3.5-9B",
      "setup_label": "Microsoft-Decision-1",
      "maker": "Microsoft",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 30.555555555555546,
      "raw": {
       "metric": "accuracy",
       "value": 0.4444444444444444,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.08000077411293488,
      "improvement_points": null,
      "runs": [
       "logic:ms-decision-1_en",
       "logic:ms-decision-1_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "Valen-Team/Valen-Decision-1.5-9B",
      "commit": "5a512117ebc0f2b69f311becfcf47cdef8abf568",
      "model_key": "valen-decision-1.5-9b",
      "name": "Valen-Decision-1.5-9B",
      "display_name": "Valen-Decision-1.5-9B",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": "Qwen3.5-9B",
      "setup_label": "Valen-Decision-1.5-9B",
      "maker": "Valen-Team",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 30.44444444444444,
      "raw": {
       "metric": "accuracy",
       "value": 0.44355555555555554,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.05821593079308574,
      "improvement_points": null,
      "runs": [
       "logic:valen-9b_en",
       "logic:valen-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.019166666666666665,
      "gpu_cost_usd": 0.047725000000000004
     },
     {
      "repo": "PelaAI/KnowLine-9B-Gen1",
      "commit": "325378d128e9f5eb5088196668f5874c5682b6c0",
      "model_key": "knowline-9b-gen1",
      "name": "KnowLine-9B-Gen1",
      "display_name": "KnowLine-9B-Gen1",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": "Qwen3.5-9B",
      "setup_label": "KnowLine-9B-Gen1",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 30.33333333333333,
      "raw": {
       "metric": "accuracy",
       "value": 0.44266666666666665,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.03528201672992203,
      "improvement_points": null,
      "runs": [
       "logic:knowline-9b_en",
       "logic:knowline-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.01638888888888889,
      "gpu_cost_usd": 0.04080833333333334
     },
     {
      "repo": "scar-ai/vera-medium-omni",
      "commit": "1f23a6cf5304e8e53b7ded53980765d5f8f4047c",
      "model_key": "vera-medium-omni",
      "name": "vera-medium-omni",
      "display_name": "vera-medium-omni",
      "tag": "",
      "class_label": "Open · 4.2B",
      "base_model": "Qwen3.5-4B",
      "setup_label": "vera-medium-omni",
      "maker": "scar-ai",
      "kind": "decision",
      "params_total": 4.2,
      "params_active": 4.2,
      "quantization": "BF16",
      "score_0_100": 30.111111111111104,
      "raw": {
       "metric": "accuracy",
       "value": 0.4408888888888889,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.04679093333333333,
      "improvement_points": null,
      "runs": [
       "logic:vera-medium-omni_en",
       "logic:vera-medium-omni_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.011666666666666667,
      "gpu_cost_usd": 0.029050000000000003
     },
     {
      "repo": "PelaAI/KnowLine-9B-Gen2",
      "commit": "8ec8eb609bc10aa6050ae3754aee419d458fae42",
      "model_key": "knowline-9b-gen2",
      "name": "KnowLine-9B-Gen2",
      "display_name": "KnowLine-9B-Gen2",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": "Qwen3.5-9B",
      "setup_label": "KnowLine-9B-Gen2",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 29.888888888888886,
      "raw": {
       "metric": "accuracy",
       "value": 0.4391111111111111,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.03240833454261845,
      "improvement_points": null,
      "runs": [
       "logic:knowline-9b-gen2_en",
       "logic:knowline-9b-gen2_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.018055555555555554,
      "gpu_cost_usd": 0.044958333333333336
     },
     {
      "repo": "apertixai/cerno-9b-v1.2",
      "commit": "f02a632fbe3a4698cd1ff5c0b929ea240e2b8eda",
      "model_key": "cerno-9b-v1.2",
      "name": "Cerno 9B v1.2",
      "display_name": "Cerno 9B v1.2",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": null,
      "setup_label": "Cerno 9B v1.2",
      "maker": "Apertix AI",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 29.111111111111104,
      "raw": {
       "metric": "accuracy",
       "value": 0.4328888888888889,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.16968382222222225,
      "improvement_points": null,
      "runs": [
       "logic:cerno-9b_en",
       "logic:cerno-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.07055555555555555,
      "gpu_cost_usd": 0.015015479951527539
     },
     {
      "repo": "perplexity-ai/pplx-decider-flash-v1.2",
      "commit": "6d771ad7b5ecff5e93b8fc7b90d4c5ccc341954d",
      "model_key": "perplexity-decider-flash-v1.2",
      "name": "Perplexity Decider Flash v1.2",
      "display_name": "Perplexity Decider Flash v1.2",
      "tag": "",
      "class_label": "Open · 4.0B",
      "base_model": null,
      "setup_label": "Perplexity Decider Flash v1.2",
      "maker": "Perplexity",
      "kind": "decision",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "BF16",
      "score_0_100": 28.888888888888886,
      "raw": {
       "metric": "accuracy",
       "value": 0.4311111111111111,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.059703545385305964,
      "improvement_points": null,
      "runs": [
       "logic:decider-flash-v1.2_en",
       "logic:decider-flash-v1.2_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.015555555555555555,
      "gpu_cost_usd": 0.038733333333333335
     },
     {
      "repo": "ZefanCai/Open-Jev-27B-v1.1",
      "commit": "28cf73067d5b337860bbef3c85b8b82ba8730956",
      "model_key": "open-jev-27b-v1.1",
      "name": "Open-Jev-27B-v1.1",
      "display_name": "Open-Jev-27B-v1.1",
      "tag": "",
      "class_label": "Open · 27B",
      "base_model": "Qwen3.8-27B + adapter",
      "setup_label": "Open-Jev 27B v1.1",
      "maker": "Zefan Cai",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 28.222222222222214,
      "raw": {
       "metric": "accuracy",
       "value": 0.42577777777777776,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.12899827835501443,
      "improvement_points": null,
      "runs": [
       "logic:open-jev_en",
       "logic:open-jev_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); the repository holds an adapter and head on Qwen3.8-27B",
      "gpu_hours": 0.13305555555555557,
      "gpu_cost_usd": 0.3313083333333333
     },
     {
      "repo": "PelaAI/KnowLine-4B-Gen5.5",
      "commit": "779944d2cc8ffa19cef386aee83c83bcb967ade2",
      "model_key": "knowline-4b-gen5.5",
      "name": "KnowLine 4B Gen5.5",
      "display_name": "KnowLine 4B Gen5.5",
      "tag": "",
      "class_label": "Open · 4.0B",
      "base_model": null,
      "setup_label": "KnowLine 4B Gen5.5",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "FP8",
      "score_0_100": 27.888888888888886,
      "raw": {
       "metric": "accuracy",
       "value": 0.4231111111111111,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.13698799611531032,
      "improvement_points": null,
      "runs": [
       "logic:knowline-4b_en",
       "logic:knowline-4b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.03111111111111111,
      "gpu_cost_usd": 0.006620999033744426
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "drex-2.0",
      "name": "Drex 2.0",
      "display_name": "Drex 2.0",
      "tag": "",
      "class_label": "Open · 35B MoE · 3.0B active",
      "base_model": null,
      "setup_label": "Drex 2.0",
      "maker": "Nace.AI",
      "kind": "decision",
      "params_total": 35.0,
      "params_active": 3.0,
      "quantization": null,
      "score_0_100": 27.333333333333332,
      "raw": {
       "metric": "accuracy",
       "value": 0.4186666666666667,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.04318124444444447,
      "improvement_points": null,
      "runs": [
       "logic:drex-2.0_en",
       "logic:drex-2.0_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "Cloudflare/clef",
      "commit": "2f3de3dd85f379784083b0814d997ab627200f0c",
      "model_key": "clef",
      "name": "Clef",
      "display_name": "Clef",
      "tag": "",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Clef",
      "maker": "Cloudflare",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 25.999999999999996,
      "raw": {
       "metric": "accuracy",
       "value": 0.408,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.1570599111111111,
      "improvement_points": null,
      "runs": [
       "logic:clef_en",
       "logic:clef_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.0636111111111111,
      "gpu_cost_usd": 0.15839166666666668
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "liquid-d1",
      "name": "d1",
      "display_name": "d1",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Liquid d1",
      "maker": "Liquid AI",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 25.999999999999996,
      "raw": {
       "metric": "accuracy",
       "value": 0.408,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.10909698559044553,
      "improvement_points": null,
      "runs": [
       "logic:d1_en",
       "logic:d1_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "OmniJev/OneJev-27B",
      "commit": "a5e5f55bdbdc9deda44f3fcb5b163776b9b369e8",
      "model_key": "onejev-27b",
      "name": "OneJev-27B",
      "display_name": "OneJev-27B",
      "tag": "",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "OneJev 27B",
      "maker": "OmniJev",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 25.77777777777777,
      "raw": {
       "metric": "accuracy",
       "value": 0.4062222222222222,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.04628716799999997,
      "improvement_points": null,
      "runs": [
       "logic:onejev_en",
       "logic:onejev_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.03888888888888889,
      "gpu_cost_usd": 0.09683333333333334
     },
     {
      "repo": "quincyqiang/chinese-decision-pointer-4b",
      "commit": "889f23e2655e5cba1208bb17f90526092813f5f2",
      "model_key": "chinese-decision-pointer-4b",
      "name": "Chinese Decision Pointer 4B",
      "display_name": "Chinese Decision Pointer 4B",
      "tag": "",
      "class_label": "Open · 4.0B",
      "base_model": "Qwen3.5-4B",
      "setup_label": "Chinese Decision Pointer 4B",
      "maker": "quincyqiang",
      "kind": "decision",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "BF16",
      "score_0_100": 25.444444444444443,
      "raw": {
       "metric": "accuracy",
       "value": 0.40355555555555556,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.15482393511855636,
      "improvement_points": null,
      "runs": [
       "logic:chinese-pointer-4b_en",
       "logic:chinese-pointer-4b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.013333333333333334,
      "gpu_cost_usd": 0.0332
     },
     {
      "repo": "victorbennett/Verdin-4B-v1.1",
      "commit": "45830d216bb7e45f43f0a482c131a7cafa6d189f",
      "model_key": "verdin-4b-v1.1",
      "name": "Verdin-4B-v1.1",
      "display_name": "Verdin-4B-v1.1",
      "tag": "",
      "class_label": "Open · 4.0B",
      "base_model": "Qwen3.5-4B",
      "setup_label": "Verdin-4B-v1.1",
      "maker": "victorbennett",
      "kind": "decision",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "BF16",
      "score_0_100": 24.66666666666666,
      "raw": {
       "metric": "accuracy",
       "value": 0.3973333333333333,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.10755777164444635,
      "improvement_points": null,
      "runs": [
       "logic:verdin-4b_en",
       "logic:verdin-4b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.015,
      "gpu_cost_usd": 0.03735
     },
     {
      "repo": "EldanRing/Winnow-12B",
      "commit": "b2b14213dfa252e6d6b543c8b334762e51772d29",
      "model_key": "winnow-12b",
      "name": "Winnow-12B",
      "display_name": "Winnow-12B",
      "tag": "",
      "class_label": "Open · 12B",
      "base_model": null,
      "setup_label": "Winnow-12B",
      "maker": "EldanRing",
      "kind": "decision",
      "params_total": 12.0,
      "params_active": 12.0,
      "quantization": "Q8_0",
      "score_0_100": 24.66666666666666,
      "raw": {
       "metric": "accuracy",
       "value": 0.3973333333333333,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.14440644531909352,
      "improvement_points": null,
      "runs": [
       "logic:winnow_en",
       "logic:winnow_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); the Q8_0 GGUF, served by winnow-inference at d4631fb",
      "gpu_hours": 0.023333333333333334,
      "gpu_cost_usd": 0.058100000000000006
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "cygnet",
      "name": "Cygnet",
      "display_name": "Cygnet",
      "tag": "",
      "class_label": "Open · 12B",
      "base_model": "Gemma 4 12B IT",
      "setup_label": "Cygnet",
      "maker": "Blockbrain Labs",
      "kind": "decision",
      "params_total": 11.95,
      "params_active": 11.95,
      "quantization": "BF16",
      "score_0_100": 24.11111111111111,
      "raw": {
       "metric": "accuracy",
       "value": 0.3928888888888889,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.11178481816625094,
      "improvement_points": null,
      "runs": [
       "logic:cygnet_en",
       "logic:cygnet_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); no weights of its own: google/gemma-4-12B-it at 707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7, unchanged, behind cygnet-recipe's shim (github.com/blockbrain-ai/cygnet-recipe 3cf591c, temperature 3.4); repo left empty so it is not matched with Gemma 4 12B",
      "gpu_hours": 0.013888888888888888,
      "gpu_cost_usd": 0.034583333333333334
     },
     {
      "repo": "Cloudflare/clef-flash",
      "commit": "fde727a287004204b7518dcc983fe64379776712",
      "model_key": "clef-flash",
      "name": "Clef-Flash",
      "display_name": "Clef-Flash",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": null,
      "setup_label": "Clef-flash",
      "maker": "Cloudflare",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 23.66666666666666,
      "raw": {
       "metric": "accuracy",
       "value": 0.3893333333333333,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.114956,
      "improvement_points": null,
      "runs": [
       "logic:clef-flash_en",
       "logic:clef-flash_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.03666666666666667,
      "gpu_cost_usd": 0.0913
     },
     {
      "repo": "fastino/GLiDE-2B",
      "commit": "00f122c120cdeed82284106cb77d9bab5a50ad16",
      "model_key": "glide-2b",
      "name": "GLiDE 2B",
      "display_name": "GLiDE 2B",
      "tag": "",
      "class_label": "Open · 1.9B",
      "base_model": null,
      "setup_label": "GLiDE 2B (open)",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": 1.88,
      "params_active": 1.88,
      "quantization": "BF16",
      "score_0_100": 15.333333333333329,
      "raw": {
       "metric": "accuracy",
       "value": 0.32266666666666666,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.16221905828461392,
      "improvement_points": null,
      "runs": [
       "logic:glide-2b_en",
       "logic:glide-2b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.011944444444444445,
      "gpu_cost_usd": 0.0025419907004554498
     },
     {
      "repo": "wayfind/metask-jev-4b-policy-mix",
      "commit": "ea20fe85b28733b1522721dec119bec50947a869",
      "model_key": "metask-jev-4b",
      "name": "Metask-Jev-4B",
      "display_name": "Metask-Jev-4B",
      "tag": "",
      "class_label": "Open · 4.5B",
      "base_model": null,
      "setup_label": "Metask-Jev 4B",
      "maker": "Metask Lab",
      "kind": "decision",
      "params_total": 4.54,
      "params_active": 4.54,
      "quantization": "BF16",
      "score_0_100": 15.111111111111109,
      "raw": {
       "metric": "accuracy",
       "value": 0.3208888888888889,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.0254769436253442,
      "improvement_points": null,
      "runs": [
       "logic:metask_en",
       "logic:metask_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.04111111111111111,
      "gpu_cost_usd": 0.10236666666666668
     },
     {
      "repo": "PelaAI/KnowLine-2B-Gen1",
      "commit": "53245d470a4ce316e32fd0d18c24e142c697f40b",
      "model_key": "knowline-2b-gen1",
      "name": "KnowLine-2B-Gen1",
      "display_name": "KnowLine-2B-Gen1",
      "tag": "",
      "class_label": "Open · 2.0B",
      "base_model": "Qwen3.5-2B",
      "setup_label": "KnowLine-2B-Gen1",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 2.0,
      "params_active": 2.0,
      "quantization": "BF16",
      "score_0_100": 14.666666666666666,
      "raw": {
       "metric": "accuracy",
       "value": 0.31733333333333336,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.08343323308142794,
      "improvement_points": null,
      "runs": [
       "logic:knowline-2b_en",
       "logic:knowline-2b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.02,
      "gpu_cost_usd": 0.004256356521692845
     },
     {
      "repo": "startlux-models/StartLux-Decision-v1.1-2B-Instant",
      "commit": "f1e6cd147dda82c9ffa3e8e725d5c1f175566fb1",
      "model_key": "startlux-decision-v1.1-2b",
      "name": "StartLux-Decision-v1.1-2B-Instant",
      "display_name": "StartLux-Decision-v1.1-2B-Instant",
      "tag": "",
      "class_label": "Open · 2.0B",
      "base_model": null,
      "setup_label": "StartLux-Decision v1.1 2B",
      "maker": "StartLux",
      "kind": "decision",
      "params_total": 2.0,
      "params_active": 2.0,
      "quantization": "BF16",
      "score_0_100": 13.444444444444445,
      "raw": {
       "metric": "accuracy",
       "value": 0.3075555555555556,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.053115119095329164,
      "improvement_points": null,
      "runs": [
       "logic:startlux-2b_en",
       "logic:startlux-2b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.013333333333333334,
      "gpu_cost_usd": 0.002837571014461897
     },
     {
      "repo": "LiquidAI/d1-2B",
      "commit": "10e7671d44e4b37700020b1eed3f594c5f2af0b4",
      "model_key": "liquid-d1-2b",
      "name": "d1-2B",
      "display_name": "d1-2B",
      "tag": "",
      "class_label": "Open · 1.9B",
      "base_model": "LFM3-VL-2B",
      "setup_label": "Liquid d1-2B (open weights)",
      "maker": "Liquid AI",
      "kind": "decision",
      "params_total": 1.92,
      "params_active": 1.92,
      "quantization": "BF16",
      "score_0_100": 13.333333333333327,
      "raw": {
       "metric": "accuracy",
       "value": 0.30666666666666664,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.06882548562528445,
      "improvement_points": null,
      "runs": [
       "logic:d1-2b_en",
       "logic:d1-2b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.006666666666666667,
      "gpu_cost_usd": 0.0166
     },
     {
      "repo": "fastino/GLiDE-0.8B",
      "commit": "9eacdeec79fa48ec52a8d6f54b331376b4bbfaf0",
      "model_key": "glide-0.8b",
      "name": "GLiDE 0.8B",
      "display_name": "GLiDE 0.8B",
      "tag": "",
      "class_label": "Open · 0.8B",
      "base_model": null,
      "setup_label": "GLiDE 0.8B (open)",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": 0.75,
      "params_active": 0.75,
      "quantization": "BF16",
      "score_0_100": 11.444444444444445,
      "raw": {
       "metric": "accuracy",
       "value": 0.29155555555555557,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.13984170771697627,
      "improvement_points": null,
      "runs": [
       "logic:glide-0.8b_en",
       "logic:glide-0.8b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.008333333333333333,
      "gpu_cost_usd": 0.0017734818840386857
     },
     {
      "repo": "OmniJev/OneJev1.6-0.8B",
      "commit": "bcd88a382f314b26cb7b1b7f369c1ed3502ee7a0",
      "model_key": "onejev1.6-0.8b",
      "name": "OneJev1.6-0.8B",
      "display_name": "OneJev1.6-0.8B",
      "tag": "",
      "class_label": "Open · 0.8B",
      "base_model": null,
      "setup_label": "OneJev1.6-0.8B",
      "maker": "OmniJev",
      "kind": "decision",
      "params_total": 0.8,
      "params_active": 0.8,
      "quantization": "BF16",
      "score_0_100": 8.777777777777773,
      "raw": {
       "metric": "accuracy",
       "value": 0.2702222222222222,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.1124625022222222,
      "improvement_points": null,
      "runs": [
       "logic:onejev1.6-0.8b_en",
       "logic:onejev1.6-0.8b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.006111111111111111,
      "gpu_cost_usd": 0.0013005533816283694
     },
     {
      "repo": "PelaAI/KnowLine-0.8B-Gen1",
      "commit": "6b27831f7b783c03278bd946d4d8f4f7a98a3798",
      "model_key": "knowline-0.8b-gen1",
      "name": "KnowLine-0.8B-Gen1",
      "display_name": "KnowLine-0.8B-Gen1",
      "tag": "",
      "class_label": "Open · 0.8B",
      "base_model": "Qwen3.5-0.8B",
      "setup_label": "KnowLine-0.8B-Gen1",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 0.8,
      "params_active": 0.8,
      "quantization": "BF16",
      "score_0_100": 8.333333333333332,
      "raw": {
       "metric": "accuracy",
       "value": 0.26666666666666666,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.08374339197455763,
      "improvement_points": null,
      "runs": [
       "logic:knowline-0.8b_en",
       "logic:knowline-0.8b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.010555555555555556,
      "gpu_cost_usd": 0.002246410386449002
     },
     {
      "repo": "startlux-models/StartLux-Decision-v1.1-0.8B-Instant",
      "commit": "6496ae641cc71b149ce9dad69bd97f2a44e7bf11",
      "model_key": "startlux-decision-v1.1-0.8b-instant",
      "name": "StartLux-Decision-v1.1-0.8B-Instant",
      "display_name": "StartLux-Decision-v1.1-0.8B-Instant",
      "tag": "",
      "class_label": "Open · 0.8B",
      "base_model": "Qwen3.5-0.8B",
      "setup_label": "StartLux-Decision-v1.1-0.8B-Instant",
      "maker": "StartLux",
      "kind": "decision",
      "params_total": 0.8,
      "params_active": 0.8,
      "quantization": "BF16",
      "score_0_100": 5.8888888888888875,
      "raw": {
       "metric": "accuracy",
       "value": 0.24711111111111111,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.07773130108783036,
      "improvement_points": null,
      "runs": [
       "logic:startlux-0.8b_en",
       "logic:startlux-0.8b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.009444444444444445,
      "gpu_cost_usd": 0.0020099461352438438
     },
     {
      "repo": "knowledgator/gliclass-v4-edge",
      "commit": "885af1af8a07edf4ed34df83ca0d995ed8201283",
      "model_key": "gliclass-v4-edge",
      "name": "GLiClass v4 Edge",
      "display_name": "GLiClass v4 Edge",
      "tag": "",
      "class_label": "Open · 0.4B",
      "base_model": "DeBERTa-v3-large",
      "setup_label": "GLiClass v4 Edge",
      "maker": null,
      "kind": "decision",
      "params_total": 0.4,
      "params_active": 0.4,
      "quantization": "BF16",
      "score_0_100": 4.444444444444442,
      "raw": {
       "metric": "accuracy",
       "value": 0.23555555555555555,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.0874588157467637,
      "improvement_points": null,
      "runs": [
       "logic:gliclass-v4-edge_en",
       "logic:gliclass-v4-edge_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.006944444444444444,
      "gpu_cost_usd": 0.001477901570032238
     },
     {
      "repo": "fastino/GLiDE-0.4B",
      "commit": "6bda7ab91aeb36dcad6d27828a8871e548c7fc71",
      "model_key": "glide-0.4b",
      "name": "GLiDE 0.4B",
      "display_name": "GLiDE 0.4B",
      "tag": "",
      "class_label": "Open · 0.4B",
      "base_model": null,
      "setup_label": "GLiDE 0.4B (open)",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": 0.4,
      "params_active": 0.4,
      "quantization": "BF16",
      "score_0_100": 4.0,
      "raw": {
       "metric": "accuracy",
       "value": 0.232,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.08792828638635584,
      "improvement_points": null,
      "runs": [
       "logic:glide-0.4b_en",
       "logic:glide-0.4b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.005,
      "gpu_cost_usd": 0.0010640891304232113
     },
     {
      "repo": "StrandsAgents/strands-decider-2B-hobson-v21",
      "commit": "2b52a6235c1b8306bbfa30b00b9d4b74b63a39f5",
      "model_key": "strands-decider-2b",
      "name": "Strands Decider 2B",
      "display_name": "Strands Decider 2B",
      "tag": "",
      "class_label": "Open · 2.0B",
      "base_model": null,
      "setup_label": "Strands Decider 2B",
      "maker": "Strands Agents",
      "kind": "decision",
      "params_total": 2.0,
      "params_active": 2.0,
      "quantization": "BF16",
      "score_0_100": 2.999999999999999,
      "raw": {
       "metric": "accuracy",
       "value": 0.224,
       "questions": 1125,
       "options": 5
      },
      "calibration_error": 0.1204904,
      "improvement_points": null,
      "runs": [
       "logic:strands_en",
       "logic:strands_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); the repository holds a LoRA and head on Qwen3.5-2B-Base",
      "gpu_hours": 0.0125,
      "gpu_cost_usd": 0.031125000000000003
     }
    ]
   },
   "sha256": "f090d59d41ff121e812b383cc9dbda6963e06618c65627ab7c64308441f0edb0"
  },
  "math": {
   "url": "https://zekiworks.github.io/analysis/logic/bench-scores-math.json",
   "published": true,
   "seen": "2026-11-16",
   "file": {
    "bench": "math",
    "title": "Math",
    "url": "https://zekiworks.github.io/analysis/logic/math/",
    "updated": "2026-11-16T21:03:26Z",
    "score": {
     "zero": "chance: 1 of 5 options right on every question (accuracy 20%)",
     "hundred": "every one of the 2,163 Math questions right (accuracy 100%)"
    },
    "models": [
     {
      "repo": null,
      "commit": null,
      "model_key": "claude-opus-5.5",
      "name": "Claude Opus 5.5",
      "display_name": "Claude Opus 5.5",
      "tag": "chat",
      "class_label": "Closed · Flagship",
      "base_model": null,
      "setup_label": "Claude Opus 5.5",
      "maker": "Anthropic",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 99.711049468331,
      "raw": {
       "metric": "accuracy",
       "value": 0.9976883957466481,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:claude-opus-5.5-sub_en",
       "math:claude-opus-5.5-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "qwen4-max",
      "name": "Qwen4-Max",
      "display_name": "Qwen4-Max",
      "tag": "high · chat",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "Qwen4-Max",
      "maker": "Alibaba",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 99.711049468331,
      "raw": {
       "metric": "accuracy",
       "value": 0.9976883957466481,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:qwen4-max_en",
       "math:qwen4-max_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "claude-fable-5.1",
      "name": "Claude Fable 5.1",
      "display_name": "Claude Fable 5.1",
      "tag": "chat",
      "class_label": "Closed · Flagship",
      "base_model": null,
      "setup_label": "Claude Fable 5.1",
      "maker": "Anthropic",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 99.65325936199723,
      "raw": {
       "metric": "accuracy",
       "value": 0.9972260748959778,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:claude-fable-5.1-sub_en",
       "math:claude-fable-5.1-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "claude-sonnet-5.5",
      "name": "Claude Sonnet 5.5",
      "display_name": "Claude Sonnet 5.5",
      "tag": "chat",
      "class_label": "Closed · Workhorse",
      "base_model": null,
      "setup_label": "Claude Sonnet 5.5",
      "maker": "Anthropic",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 99.59546925566343,
      "raw": {
       "metric": "accuracy",
       "value": 0.9967637540453075,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:claude-sonnet-5.5-sub_en",
       "math:claude-sonnet-5.5-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "deepseek-v4.1-flash-1017",
      "name": "DeepSeek-V4.1-Flash",
      "display_name": "DeepSeek-V4.1-Flash",
      "tag": "high · chat · 1017",
      "class_label": "Open · 552B MoE · 16B active",
      "base_model": null,
      "setup_label": "DeepSeek-V4.1-Flash 1017",
      "maker": "DeepSeek",
      "kind": "chat",
      "params_total": 552.0,
      "params_active": 16.0,
      "quantization": null,
      "score_0_100": 99.53767914932962,
      "raw": {
       "metric": "accuracy",
       "value": 0.996301433194637,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:deepseek-v4.1-flash-1017_en",
       "math:deepseek-v4.1-flash-1017_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "gpt-6-astra",
      "name": "GPT-6 Astra",
      "display_name": "GPT-6 Astra",
      "tag": "chat",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "GPT-6 Astra",
      "maker": "OpenAI",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 99.53767914932962,
      "raw": {
       "metric": "accuracy",
       "value": 0.996301433194637,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:gpt-6-astra-sub_en",
       "math:gpt-6-astra-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "gpt-6.1-sol",
      "name": "GPT-6.1 Sol",
      "display_name": "GPT-6.1 Sol",
      "tag": "chat",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "GPT-6.1 Sol",
      "maker": "OpenAI",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 99.53767914932962,
      "raw": {
       "metric": "accuracy",
       "value": 0.996301433194637,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:gpt-6.1-sol-sub_en",
       "math:gpt-6.1-sol-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "inkling-small",
      "name": "Inkling-Small",
      "display_name": "Inkling-Small",
      "tag": "high · chat · Jul 2026",
      "class_label": "Open · 276B MoE · 12B active",
      "base_model": null,
      "setup_label": "Inkling Small",
      "maker": "Thinking Machines",
      "kind": "chat",
      "params_total": 276.0,
      "params_active": 12.0,
      "quantization": null,
      "score_0_100": 99.53767914932962,
      "raw": {
       "metric": "accuracy",
       "value": 0.996301433194637,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:inkling-small-64k_en",
       "math:inkling-small-64k_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "gemini-4-carbon",
      "name": "Gemini 4 Carbon",
      "display_name": "Gemini 4 Carbon",
      "tag": "chat",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "Gemini 4 Carbon (Antigravity subscription)",
      "maker": "Google",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 99.47988904299584,
      "raw": {
       "metric": "accuracy",
       "value": 0.9958391123439667,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:gemini-4-carbon-sub_en",
       "math:gemini-4-carbon-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "gemini-4-hydrogen",
      "name": "Gemini 4 Hydrogen",
      "display_name": "Gemini 4 Hydrogen",
      "tag": "chat",
      "class_label": "",
      "base_model": null,
      "setup_label": "Gemini 4 Hydrogen (Antigravity subscription)",
      "maker": "Google",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 99.47988904299584,
      "raw": {
       "metric": "accuracy",
       "value": 0.9958391123439667,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:gemini-4-hydrogen-sub_en",
       "math:gemini-4-hydrogen-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "muse-ember",
      "name": "Muse Ember",
      "display_name": "Muse Ember",
      "tag": "max · chat",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Muse Ember",
      "maker": "Meta",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 99.47988904299584,
      "raw": {
       "metric": "accuracy",
       "value": 0.9958391123439667,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:muse-ember_en",
       "math:muse-ember_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "grok-4.7",
      "name": "Grok 4.7",
      "display_name": "Grok 4.7",
      "tag": "chat",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "Grok 4.7",
      "maker": "xAI",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 99.42209893666204,
      "raw": {
       "metric": "accuracy",
       "value": 0.9953767914932964,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:grok-4.7-sub_en",
       "math:grok-4.7-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "inkling-2-small",
      "name": "Inkling 2 Small",
      "display_name": "Inkling 2 Small",
      "tag": "high · chat",
      "class_label": "Open · 594B MoE · 17B active",
      "base_model": null,
      "setup_label": "Inkling 2 Small",
      "maker": "Thinking Machines",
      "kind": "chat",
      "params_total": 594.0,
      "params_active": 17.0,
      "quantization": null,
      "score_0_100": 99.42209893666204,
      "raw": {
       "metric": "accuracy",
       "value": 0.9953767914932964,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:inkling-2-small_en",
       "math:inkling-2-small_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "qwen3.8-flash",
      "name": "Qwen3.8 Flash",
      "display_name": "Qwen3.8 Flash",
      "tag": "off · chat",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "Qwen3.8 Flash",
      "maker": "Alibaba",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 99.36430883032823,
      "raw": {
       "metric": "accuracy",
       "value": 0.994914470642626,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:qwen3.8-flash_en",
       "math:qwen3.8-flash_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "mimo-v2.6-flash",
      "name": "MiMo-V2.6-Flash",
      "display_name": "MiMo-V2.6-Flash",
      "tag": "on · chat",
      "class_label": "Open · 309B MoE · 15B active",
      "base_model": null,
      "setup_label": "MiMo V2.6 Flash",
      "maker": "Xiaomi",
      "kind": "chat",
      "params_total": 309.0,
      "params_active": 15.0,
      "quantization": null,
      "score_0_100": 99.30651872399446,
      "raw": {
       "metric": "accuracy",
       "value": 0.9944521497919556,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:mimo-v2.6-flash-64k_en",
       "math:mimo-v2.6-flash-64k_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "Qwen/Qwen3.5-4B",
      "commit": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",
      "model_key": "qwen3.5-4b",
      "name": "Qwen3.5-4B",
      "display_name": "Qwen3.5-4B",
      "tag": "on · chat",
      "class_label": "Open · 4.7B",
      "base_model": null,
      "setup_label": "Qwen3.5-4B (chat, thinking, on our GPU)",
      "maker": "Alibaba",
      "kind": "chat",
      "params_total": 4.66,
      "params_active": 4.66,
      "quantization": "BF16",
      "score_0_100": 99.30651872399446,
      "raw": {
       "metric": "accuracy",
       "value": 0.9944521497919556,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:qwen3.5-4b_en",
       "math:qwen3.5-4b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: Qwen3.5-4B (off · chat)",
      "gpu_hours": 1.1041666666666667,
      "gpu_cost_usd": 2.749375
     },
     {
      "repo": "google/gemma-4-31B-it",
      "commit": "842da3794eaa0b77d5f08bae87a17459d91ff475",
      "model_key": "gemma-4-31b",
      "name": "Gemma 4 31B",
      "display_name": "Gemma 4 31B",
      "tag": "chat",
      "class_label": "Open · 31B",
      "base_model": null,
      "setup_label": "Gemma 4 31B (chat, on our GPU)",
      "maker": "Google",
      "kind": "chat",
      "params_total": 30.7,
      "params_active": 30.7,
      "quantization": "FP8",
      "score_0_100": 99.24872861766065,
      "raw": {
       "metric": "accuracy",
       "value": 0.9939898289412853,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:gemma-4-31b-it_en_t0",
       "math:gemma-4-31b-it_en_t0_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); BF16 weights quantized to FP8 by vLLM at load",
      "gpu_hours": 0.46111111111111114,
      "gpu_cost_usd": 1.1481666666666668
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "deepseek-v4-flash-dspark",
      "name": "DeepSeek-V4-Flash",
      "display_name": "DeepSeek-V4-Flash",
      "tag": "chat",
      "class_label": "Open · 285B MoE · 13B active",
      "base_model": null,
      "setup_label": "deepseek-v4-flash-dspark",
      "maker": "DeepSeek",
      "kind": "chat",
      "params_total": 285.0,
      "params_active": 13.0,
      "quantization": null,
      "score_0_100": 99.13314840499307,
      "raw": {
       "metric": "accuracy",
       "value": 0.9930651872399445,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:deepseek-v4-flash_en_t0"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10); no complete 'I don't know' run",
      "gpu_hours": 2.9963888888888888,
      "gpu_cost_usd": 0.6376849694375101
     },
     {
      "repo": "openbmb/MiniCPM5-2B",
      "commit": "42433e90dad95123ab87729fd7011e7110629556",
      "model_key": "minicpm5-2b",
      "name": "MiniCPM5-2B",
      "display_name": "MiniCPM5-2B",
      "tag": "on · chat",
      "class_label": "Open · 2.5B",
      "base_model": null,
      "setup_label": "MiniCPM5-2B (chat, thinking, on our GPU)",
      "maker": "OpenBMB",
      "kind": "chat",
      "params_total": 2.52,
      "params_active": 2.52,
      "quantization": "BF16",
      "score_0_100": 99.0368315611034,
      "raw": {
       "metric": "accuracy",
       "value": 0.9922946524888273,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:minicpm5-2b_en",
       "math:minicpm5-2b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: MiniCPM5-2B (off · chat); accuracy is the mean of 3 seed runs (min 0.9917, max 0.9926): seed 0 minicpm5-2b_en 2147/2163, seed 1 minicpm5-2b-seed1_en 2145/2163, seed 2 minicpm5-2b-seed2_en 2147/2163",
      "gpu_hours": 0.3605555555555556,
      "gpu_cost_usd": 0.8977833333333335
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "gpt-6.1-terra",
      "name": "GPT-6.1 Terra",
      "display_name": "GPT-6.1 Terra",
      "tag": "chat",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "GPT-6.1 Terra",
      "maker": "OpenAI",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 99.01756819232546,
      "raw": {
       "metric": "accuracy",
       "value": 0.9921405455386038,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:gpt-6.1-terra-sub_en",
       "math:gpt-6.1-terra-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "ibm-granite/granite-4.2-3b",
      "commit": "4e9c275e232f10211ec88644d106a1f6a1b60e06",
      "model_key": "granite-4.2-3b",
      "name": "Granite-4.2-3B",
      "display_name": "Granite-4.2-3B",
      "tag": "high · chat",
      "class_label": "Open · 3.0B",
      "base_model": "Granite-4.1-3B-Base",
      "setup_label": "Granite-4.2-3B (chat, thinking, on our GPU)",
      "maker": "IBM",
      "kind": "chat",
      "params_total": 3.0,
      "params_active": 3.0,
      "quantization": "BF16",
      "score_0_100": 99.01756819232546,
      "raw": {
       "metric": "accuracy",
       "value": 0.9921405455386038,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:granite-4.2-3b-64k_en",
       "math:granite-4.2-3b-64k_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.6783333333333333,
      "gpu_cost_usd": 1.6890500000000002
     },
     {
      "repo": "Qwen/Qwen3.8-27B",
      "commit": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
      "model_key": "qwen3.8-27b",
      "name": "Qwen3.8-27B",
      "display_name": "Qwen3.8-27B",
      "tag": "low · chat",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Qwen3.8-27B (chat, reasoning low, on our GPU)",
      "maker": "Alibaba",
      "kind": "chat",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 99.01756819232546,
      "raw": {
       "metric": "accuracy",
       "value": 0.9921405455386038,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:qwen3.8-27b-bf16_en_low",
       "math:qwen3.8-27b-bf16_en_low_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.4602777777777778,
      "gpu_cost_usd": 1.1460916666666667
     },
     {
      "repo": "LiquidAI/LFM2.5-24B-A2B",
      "commit": "1769516ac588c4c558089c94f27d96b0dc67b183",
      "model_key": "lfm2.5-24b-a2b",
      "name": "LFM2.5-24B-A2B",
      "display_name": "LFM2.5-24B-A2B",
      "tag": "on · chat",
      "class_label": "Open · 24B MoE · 2.5B active",
      "base_model": null,
      "setup_label": "LFM2.5-24B-A2B (chat, thinking, on our GPU)",
      "maker": "Liquid AI",
      "kind": "chat",
      "params_total": 24.0,
      "params_active": 2.45,
      "quantization": "BF16",
      "score_0_100": 98.7864077669903,
      "raw": {
       "metric": "accuracy",
       "value": 0.9902912621359223,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:lfm2.5-24b-a2b_en",
       "math:lfm2.5-24b-a2b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.8158333333333333,
      "gpu_cost_usd": 2.0314250000000005
     },
     {
      "repo": "LiquidAI/LFM3-1.7B",
      "commit": "0dc4268b561499c648c7ba7c9466bf3d8d01517a",
      "model_key": "lfm3-1.7b",
      "name": "LFM3-1.7B",
      "display_name": "LFM3-1.7B",
      "tag": "on · chat",
      "class_label": "Open · 1.7B",
      "base_model": null,
      "setup_label": "LFM3-1.7B (chat, thinking, on our GPU)",
      "maker": null,
      "kind": "chat",
      "params_total": 1.73,
      "params_active": 1.73,
      "quantization": "BF16",
      "score_0_100": 98.67082755432268,
      "raw": {
       "metric": "accuracy",
       "value": 0.9893666204345816,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:lfm3-1.7b-vllm_en",
       "math:lfm3-1.7b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: LFM3-1.7B (off · chat), LFM3-1.7B (off · chat · NVIDIA RTX PRO 6000 Blackwell); accuracy is the mean of 3 seed runs (min 0.9884, max 0.9908): seed 0 lfm3-1.7b-vllm_en 2139/2163, seed 1 lfm3-1.7b-vllm-seed1_en 2143/2163, seed 2 lfm3-1.7b-vllm-seed2_en 2138/2163",
      "gpu_hours": 0.11277777777777778,
      "gpu_cost_usd": 0.28081666666666666
     },
     {
      "repo": "google/gemma-4-12B-it",
      "commit": "43c9d014b231474fa7212dbb48a762566c2b255d",
      "model_key": "gemma-4-12b",
      "name": "Gemma 4 12B Unified",
      "display_name": "Gemma 4 12B Unified",
      "tag": "chat",
      "class_label": "Open · 12B",
      "base_model": null,
      "setup_label": "Gemma 4 12B (chat, on our GPU)",
      "maker": "Google",
      "kind": "chat",
      "params_total": 11.95,
      "params_active": 11.95,
      "quantization": "BF16",
      "score_0_100": 98.5552473416551,
      "raw": {
       "metric": "accuracy",
       "value": 0.9884419787332409,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:gemma-4-12b-it_en",
       "math:gemma-4-12b-it_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.4058333333333333,
      "gpu_cost_usd": 1.0105250000000001
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "glm-5.3-flash",
      "name": "GLM-5.3-Flash",
      "display_name": "GLM-5.3-Flash",
      "tag": "high · chat",
      "class_label": "Open · 320B MoE · 18B active",
      "base_model": null,
      "setup_label": "GLM-5.3-Flash",
      "maker": "Z.ai",
      "kind": "chat",
      "params_total": 320.0,
      "params_active": 18.0,
      "quantization": null,
      "score_0_100": 98.49745723532129,
      "raw": {
       "metric": "accuracy",
       "value": 0.9879796578825705,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:glm-5.3-flash_en",
       "math:glm-5.3-flash_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "minimax-m3.1-flash",
      "name": "MiniMax-M3.1-Flash",
      "display_name": "MiniMax-M3.1-Flash",
      "tag": "high · chat",
      "class_label": "Open · 428B MoE · 23B active",
      "base_model": null,
      "setup_label": "MiniMax-M3.1-Flash",
      "maker": "MiniMax",
      "kind": "chat",
      "params_total": 428.0,
      "params_active": 23.0,
      "quantization": null,
      "score_0_100": 97.80397595931575,
      "raw": {
       "metric": "accuracy",
       "value": 0.9824318076745261,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:minimax-m3.1-flash_en",
       "math:minimax-m3.1-flash_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "google/gemma-4-E4B-it",
      "commit": "4ecc2ba63bcac23ba4a767fc87025b1450f84c9c",
      "model_key": "gemma-4-e4b",
      "name": "Gemma 4 E4B",
      "display_name": "Gemma 4 E4B",
      "tag": "chat",
      "class_label": "Open · 8.0B",
      "base_model": null,
      "setup_label": "Gemma 4 E4B (chat, on our GPU)",
      "maker": "Google",
      "kind": "chat",
      "params_total": 8.0,
      "params_active": 8.0,
      "quantization": "BF16",
      "score_0_100": 97.63060564031436,
      "raw": {
       "metric": "accuracy",
       "value": 0.981044845122515,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:gemma-4-e4b-it_en",
       "math:gemma-4-e4b-it_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: Gemma 4 E4B (on · chat), Gemma 4 E4B (on · chat · 32k cap), Gemma 4 E4B (on · chat · 32k cap · sampled); 4.5B effective parameters",
      "gpu_hours": 0.23555555555555555,
      "gpu_cost_usd": 0.5865333333333334
     },
     {
      "repo": "google/gemma-4-E2B-it",
      "commit": "4adf15748de9e1a0097b6274cee9c15836c41c37",
      "model_key": "gemma-4-e2b",
      "name": "Gemma 4 E2B",
      "display_name": "Gemma 4 E2B",
      "tag": "chat",
      "class_label": "Open · 5.1B",
      "base_model": null,
      "setup_label": "Gemma 4 E2B (chat, on our GPU)",
      "maker": "Google",
      "kind": "chat",
      "params_total": 5.1,
      "params_active": 5.1,
      "quantization": "BF16",
      "score_0_100": 95.9546925566343,
      "raw": {
       "metric": "accuracy",
       "value": 0.9676375404530745,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:gemma-4-e2b-it_en",
       "math:gemma-4-e2b-it_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: Gemma 4 E2B (on · chat), Gemma 4 E2B (on · chat · 32k cap), Gemma 4 E2B (on · chat · 32k cap · sampled); 2.3B effective parameters",
      "gpu_hours": 0.18083333333333335,
      "gpu_cost_usd": 0.4502750000000001
     },
     {
      "repo": "LiquidAI/LFM2.5-8B-A1B",
      "commit": "dd83f5cb02aa2938b66bba9a9998d38d0517c1f3",
      "model_key": "lfm2.5-8b-a1b",
      "name": "LFM2.5-8B-A1B",
      "display_name": "LFM2.5-8B-A1B",
      "tag": "on · chat",
      "class_label": "Open · 8.3B MoE · 1.5B active",
      "base_model": null,
      "setup_label": "LFM2.5-8B-A1B (chat, thinking, on our GPU)",
      "maker": "Liquid AI",
      "kind": "chat",
      "params_total": 8.3,
      "params_active": 1.5,
      "quantization": "BF16",
      "score_0_100": 95.31900138696255,
      "raw": {
       "metric": "accuracy",
       "value": 0.9625520110957004,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:lfm2.5-8b-a1b_en",
       "math:lfm2.5-8b-a1b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.41888888888888887,
      "gpu_cost_usd": 1.0430333333333335
     },
     {
      "repo": "Qwen/Qwen3-14B",
      "commit": "40c069824f4251a91eefaf281ebe4c544efd3e18",
      "model_key": "qwen3-14b",
      "name": "Qwen3-14B",
      "display_name": "Qwen3-14B",
      "tag": "off · chat",
      "class_label": "Open · 15B",
      "base_model": null,
      "setup_label": "Qwen3-14B (chat, reasoning off, on our GPU)",
      "maker": "Alibaba",
      "kind": "chat",
      "params_total": 14.8,
      "params_active": 14.8,
      "quantization": "BF16",
      "score_0_100": 94.7411003236246,
      "raw": {
       "metric": "accuracy",
       "value": 0.9579288025889967,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:qwen3-14b_en",
       "math:qwen3-14b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.30833333333333335,
      "gpu_cost_usd": 0.76775
     },
     {
      "repo": "Qwen/Qwen3.5-2B",
      "commit": "15852e8c16360a2fea060d615a32b45270f8a8fc",
      "model_key": "qwen3.5-2b",
      "name": "Qwen3.5-2B",
      "display_name": "Qwen3.5-2B",
      "tag": "off · chat",
      "class_label": "Open · 2.0B",
      "base_model": null,
      "setup_label": "Qwen3.5-2B (chat, thinking off, on our GPU)",
      "maker": "Alibaba",
      "kind": "chat",
      "params_total": 2.0,
      "params_active": 2.0,
      "quantization": "BF16",
      "score_0_100": 93.29634766527968,
      "raw": {
       "metric": "accuracy",
       "value": 0.9463707813222376,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:qwen3.5-2b-nothink_en",
       "math:qwen3.5-2b-nothink_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: Qwen3.5-2B (on · chat)",
      "gpu_hours": 0.31555555555555553,
      "gpu_cost_usd": 0.7857333333333334
     },
     {
      "repo": "LiquidAI/LFM2.5-2.6B",
      "commit": "7141b56cd4650af95c7c7b86983443aed050c96a",
      "model_key": "lfm2.5-2.6b",
      "name": "LFM2.5-2.6B",
      "display_name": "LFM2.5-2.6B",
      "tag": "on · chat",
      "class_label": "Open · 2.7B",
      "base_model": null,
      "setup_label": "LFM2.5-2.6B (chat, thinking, on our GPU)",
      "maker": "Liquid AI",
      "kind": "chat",
      "params_total": 2.69,
      "params_active": 2.69,
      "quantization": "BF16",
      "score_0_100": 92.48728617660656,
      "raw": {
       "metric": "accuracy",
       "value": 0.9398982894128525,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:lfm2.5-2.6b_en",
       "math:lfm2.5-2.6b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.26055555555555554,
      "gpu_cost_usd": 0.6487833333333334
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "gpt-6-luna",
      "name": "GPT-6 Luna",
      "display_name": "GPT-6 Luna",
      "tag": "chat",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "GPT-6 Luna (low)",
      "maker": "OpenAI",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 92.08275543226998,
      "raw": {
       "metric": "accuracy",
       "value": 0.93666204345816,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:gpt-6-luna-low-sub_en",
       "math:gpt-6-luna-low-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "ecloudtech/Erk-14B",
      "commit": "8584b14a829d5cc803bc227f44e9012e36e8ad04",
      "model_key": "erk-14b",
      "name": "Erk-14B",
      "display_name": "Erk-14B",
      "tag": "off · chat",
      "class_label": "Open · 15B",
      "base_model": null,
      "setup_label": "Erk-14B (chat, reasoning off, on our GPU)",
      "maker": "eCloud Technology",
      "kind": "chat",
      "params_total": 14.77,
      "params_active": 14.77,
      "quantization": "BF16",
      "score_0_100": 88.44197873324086,
      "raw": {
       "metric": "accuracy",
       "value": 0.907535829865927,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:erk-14b_en",
       "math:erk-14b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.2311111111111111,
      "gpu_cost_usd": 0.5754666666666668
     },
     {
      "repo": "KRAFTON/Raon-1-nano",
      "commit": "fcf2d1d55e84e2f9fbe7b5d724dcfc4d861cfcee",
      "model_key": "raon-1-nano",
      "name": "Raon-1 nano",
      "display_name": "Raon-1 nano",
      "tag": "off · chat",
      "class_label": "Open · 3.1B",
      "base_model": "Raon-1 mini preview (MoE-to-Dense pruning and distillation)",
      "setup_label": "Raon-1 nano (chat, thinking off, KRAFTON vLLM fork, on our GPU)",
      "maker": "KRAFTON",
      "kind": "chat",
      "params_total": 3.1,
      "params_active": 3.1,
      "quantization": null,
      "score_0_100": 86.93943596856218,
      "raw": {
       "metric": "accuracy",
       "value": 0.8955154877484974,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "math:raon-1-nano-nothink_en"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: Raon-1 nano (on · chat); 1.5B effective parameters; no complete 'I don't know' run",
      "gpu_hours": 1.6511111111111112,
      "gpu_cost_usd": 4.111266666666667
     },
     {
      "repo": "MachinePulseAI/karpo-decision-31B",
      "commit": "d69cdd57eab23271cbe65236ca8a00d80d7bb8c7",
      "model_key": "karpo-decision-31b",
      "name": "Karpo Decision 31B",
      "display_name": "Karpo Decision 31B",
      "tag": "",
      "class_label": "Open · 31B",
      "base_model": null,
      "setup_label": "karpo-decision-31B",
      "maker": "MachinePulse",
      "kind": "decision",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "BF16",
      "score_0_100": 76.94174757281552,
      "raw": {
       "metric": "accuracy",
       "value": 0.8155339805825242,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.012315314621813527,
      "improvement_points": null,
      "runs": [
       "math:karpo-decision-31b_en",
       "math:karpo-decision-31b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.09777777777777778,
      "gpu_cost_usd": 0.24346666666666666
     },
     {
      "repo": "PelaAI/KnowLine-31B-Gen2",
      "commit": "fd99a9ed2255d8a67dd47a44cc429e638f611593",
      "model_key": "knowline-31b-gen2",
      "name": "KnowLine-31B-Gen2",
      "display_name": "KnowLine-31B-Gen2",
      "tag": "",
      "class_label": "Open · 31B",
      "base_model": "Gemma 4 31B",
      "setup_label": "KnowLine-31B-Gen2",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "BF16",
      "score_0_100": 75.32362459546927,
      "raw": {
       "metric": "accuracy",
       "value": 0.8025889967637541,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.03631832690906286,
      "improvement_points": null,
      "runs": [
       "math:knowline-31b-gen2_en",
       "math:knowline-31b-gen2_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.07222222222222222,
      "gpu_cost_usd": 0.17983333333333335
     },
     {
      "repo": "decisionmodels/REAX-1.3",
      "commit": "fa4bf7ee3bb56a8bc421dad090631be1c5845c2d",
      "model_key": "reax-1.3",
      "name": "REAX 1.3",
      "display_name": "REAX 1.3",
      "tag": "",
      "class_label": "Open · 57B",
      "base_model": null,
      "setup_label": "REAX 1.3",
      "maker": "REAX",
      "kind": "decision",
      "params_total": 57.36,
      "params_active": 57.36,
      "quantization": "FP8",
      "score_0_100": 72.49190938511326,
      "raw": {
       "metric": "accuracy",
       "value": 0.7799352750809061,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.07101455985854778,
      "improvement_points": null,
      "runs": [
       "math:reax13_en",
       "math:reax13_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); weights byte-identical to decisionmodels/REAX-1.3 at this commit; served by the REAX engine (github.com/reaxlabs/reax-engine ff8f595)",
      "gpu_hours": 0.12333333333333334,
      "gpu_cost_usd": 0.30710000000000004
     },
     {
      "repo": "PelaAI/KnowLine-31B-Gen1.5",
      "commit": "8e56aa1b236a6dd1064dcfd22ba1e5e360e4c140",
      "model_key": "knowline-31b-gen1.5",
      "name": "KnowLine-31B-Gen1.5",
      "display_name": "KnowLine-31B-Gen1.5",
      "tag": "",
      "class_label": "Open · 31B",
      "base_model": null,
      "setup_label": "KnowLine 31B Gen1.5",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "BF16",
      "score_0_100": 71.74063800277392,
      "raw": {
       "metric": "accuracy",
       "value": 0.7739251040221914,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.03555228644547992,
      "improvement_points": null,
      "runs": [
       "math:knowline-31b_en",
       "math:knowline-31b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.47638888888888886,
      "gpu_cost_usd": 0.10138404770421153
     },
     {
      "repo": "Poindexter-Labs/Copernicus-31B",
      "commit": "be715259e97a743341f2d7a3e6a6298dab23ca0e",
      "model_key": "copernicus-31b",
      "name": "Copernicus-31B",
      "display_name": "Copernicus-31B",
      "tag": "",
      "class_label": "Open · 31B",
      "base_model": null,
      "setup_label": "Copernicus-31B",
      "maker": "Poindexter Labs",
      "kind": "decision",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "BF16",
      "score_0_100": 71.68284789644014,
      "raw": {
       "metric": "accuracy",
       "value": 0.7734627831715211,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.06453940462900969,
      "improvement_points": null,
      "runs": [
       "math:copernicus-31b_en",
       "math:copernicus-31b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.07333333333333333,
      "gpu_cost_usd": 0.1826
     },
     {
      "repo": "sthanika-ai/Sieve-31B-v3",
      "commit": "8fcee858f7f20a50bfa3c8f984c22adb3cbb46a2",
      "model_key": "sieve-31b-v3",
      "name": "Sieve 31B v3",
      "display_name": "Sieve 31B v3",
      "tag": "",
      "class_label": "Open · 31B",
      "base_model": null,
      "setup_label": "Sieve 31B v3",
      "maker": "Sthānika AI",
      "kind": "decision",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "BF16",
      "score_0_100": 71.39389736477114,
      "raw": {
       "metric": "accuracy",
       "value": 0.7711511789181692,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.042755524734165495,
      "improvement_points": null,
      "runs": [
       "math:sieve-31b_en",
       "math:sieve-31b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.3977777777777778,
      "gpu_cost_usd": 0.0846542019314466
     },
     {
      "repo": "logitshq/Logits-Sigma-1-Duo",
      "commit": "a0c1b6e8242916aafdcf9a5a49c1a952cb56e091",
      "model_key": "logits-sigma-1-duo",
      "name": "Logits Sigma-1 Duo",
      "display_name": "Logits Sigma-1 Duo",
      "tag": "",
      "class_label": "Open · 57B",
      "base_model": null,
      "setup_label": "Logits Sigma-1 Duo",
      "maker": "Logits",
      "kind": "decision",
      "params_total": 57.36,
      "params_active": 57.36,
      "quantization": "FP8",
      "score_0_100": 69.02450300508552,
      "raw": {
       "metric": "accuracy",
       "value": 0.7521960240406842,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.07541269945078859,
      "improvement_points": null,
      "runs": [
       "math:logits-sigma-1-duo_en",
       "math:logits-sigma-1-duo_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.12222222222222222,
      "gpu_cost_usd": 0.30433333333333334
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "liquid-d1-ultra",
      "name": "d1-ultra",
      "display_name": "d1-ultra",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Liquid d1-ultra",
      "maker": "Liquid AI",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 67.63754045307444,
      "raw": {
       "metric": "accuracy",
       "value": 0.7411003236245954,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.02258235015511772,
      "improvement_points": null,
      "runs": [
       "math:d1-ultra_en",
       "math:d1-ultra_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "baselabs/mammoth-1",
      "commit": "61cec44c281c693992e3d6ebeb44dd3402dfa186",
      "model_key": "mammoth-1",
      "name": "Mammoth-1",
      "display_name": "Mammoth-1",
      "tag": "",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Mammoth-1",
      "maker": "Base Labs",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 67.46417013407304,
      "raw": {
       "metric": "accuracy",
       "value": 0.7397133610725843,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.029765094313453477,
      "improvement_points": null,
      "runs": [
       "math:mammoth-1_en",
       "math:mammoth-1_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.09027777777777778,
      "gpu_cost_usd": 0.2247916666666667
     },
     {
      "repo": "perplexity-ai/pplx-decider-v1.3",
      "commit": "9c2ebb27470d3dec2ad7ab727727e40d142c7213",
      "model_key": "perplexity-decider-27b-v1.3",
      "name": "Perplexity Decider 27B v1.3",
      "display_name": "Perplexity Decider 27B v1.3",
      "tag": "NVIDIA RTX PRO 6000 Blackwell",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Perplexity Decider 27B v1.3 (local GPU)",
      "maker": "Perplexity",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 65.61488673139159,
      "raw": {
       "metric": "accuracy",
       "value": 0.7249190938511327,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.04459839344013791,
      "improvement_points": null,
      "runs": [
       "math:decider-v1.3-local_en",
       "math:decider-v1.3-local_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.05722222222222222,
      "gpu_cost_usd": 0.14248333333333335
     },
     {
      "repo": "fastino/GLiDE-27B",
      "commit": "4d32743ae96b74f0116f050d37742bbd43c04e7d",
      "model_key": "glide-27b",
      "name": "GLiDE 27B",
      "display_name": "GLiDE 27B",
      "tag": "",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "GLiDE 27B (open)",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 64.97919556171982,
      "raw": {
       "metric": "accuracy",
       "value": 0.7198335644937587,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.04077789772290079,
      "improvement_points": null,
      "runs": [
       "math:glide-27b_en",
       "math:glide-27b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.2658333333333333,
      "gpu_cost_usd": 0.05657407210083407
     },
     {
      "repo": "vllm-sr/d3-ultra",
      "commit": "52ba9e7a00402f9560c7fba8c039608e7ddee7fb",
      "model_key": "vllm-sr-d3-ultra",
      "name": "d3-ultra",
      "display_name": "d3-ultra",
      "tag": "",
      "class_label": "Open · 26B",
      "base_model": null,
      "setup_label": "vLLM-SR d3-ultra",
      "maker": "vLLM Semantic Router",
      "kind": "decision",
      "params_total": 26.09,
      "params_active": 26.09,
      "quantization": "BF16",
      "score_0_100": 64.40129449838189,
      "raw": {
       "metric": "accuracy",
       "value": 0.7152103559870551,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.08639034384095057,
      "improvement_points": null,
      "runs": [
       "math:d3-ultra_en",
       "math:d3-ultra_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.07694444444444444,
      "gpu_cost_usd": 0.19159166666666666
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "gemini-decision-1",
      "name": "Gemini Decision 1",
      "display_name": "Gemini Decision 1",
      "tag": "20261111",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Gemini Decision 1",
      "maker": "Google",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 64.1701340730467,
      "raw": {
       "metric": "accuracy",
       "value": 0.7133610725843735,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.024586229897428504,
      "improvement_points": null,
      "runs": [
       "math:gemini-decision-1_en",
       "math:gemini-decision-1_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "fastino-glide",
      "name": "GLiDE",
      "display_name": "GLiDE",
      "tag": "API",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Fastino GLiDE",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 63.47665279704115,
      "raw": {
       "metric": "accuracy",
       "value": 0.7078132223763292,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.05037002712663995,
      "improvement_points": null,
      "runs": [
       "math:glide_en",
       "math:glide_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "vllm-sr/d3",
      "commit": "20f37b9a892acc13d4b07a4f3c4f11b2518c615c",
      "model_key": "vllm-sr-d3",
      "name": "d3",
      "display_name": "d3",
      "tag": "",
      "class_label": "Open · 26B",
      "base_model": null,
      "setup_label": "vLLM-SR d3",
      "maker": "vLLM Semantic Router",
      "kind": "decision",
      "params_total": 26.09,
      "params_active": 26.09,
      "quantization": "BF16",
      "score_0_100": 63.01433194637076,
      "raw": {
       "metric": "accuracy",
       "value": 0.7041146555709662,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.04492932511844977,
      "improvement_points": null,
      "runs": [
       "math:d3_en",
       "math:d3_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.035277777777777776,
      "gpu_cost_usd": 0.08784166666666667
     },
     {
      "repo": "nace-ai/drex-max-v1",
      "commit": "cac65e1982a0237e996419ba559004522d9a33ac",
      "model_key": "drex-max-v1",
      "name": "Drex Max",
      "display_name": "Drex Max",
      "tag": "",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Drex Max v1",
      "maker": "Nace.AI",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 62.898751733703186,
      "raw": {
       "metric": "accuracy",
       "value": 0.7031900138696255,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.11814199425449926,
      "improvement_points": null,
      "runs": [
       "math:drex-max-v1_en",
       "math:drex-max-v1_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); served with Nace's drex-decision-models code at e206b27",
      "gpu_hours": 0.0475,
      "gpu_cost_usd": 0.118275
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "drex-max-1.1",
      "name": "Drex Max v1.1",
      "display_name": "Drex Max v1.1",
      "tag": "",
      "class_label": "Closed · Efficient (provisional)",
      "base_model": null,
      "setup_label": "Drex Max 1.1",
      "maker": "Nace.AI",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 62.60980120203421,
      "raw": {
       "metric": "accuracy",
       "value": 0.7008784096162737,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.056972445603077154,
      "improvement_points": null,
      "runs": [
       "math:drex-max-1.1_en",
       "math:drex-max-1.1_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "celeris-1-decision",
      "name": "celeris-1-decision",
      "display_name": "celeris-1-decision",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Celeris-1 Decision",
      "maker": "Celeris",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 62.49422098936661,
      "raw": {
       "metric": "accuracy",
       "value": 0.6999537679149329,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.04310679611650484,
      "improvement_points": null,
      "runs": [
       "math:celeris_en",
       "math:celeris_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "fastino/GLiDE",
      "commit": "a7e45896be7c96ba3354891da790c5ec99c30b6c",
      "model_key": "fastino-glide-v1.2",
      "name": "GLiDE",
      "display_name": "GLiDE",
      "tag": "v1.2",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Fastino GLiDE v1.2",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 62.320850670365225,
      "raw": {
       "metric": "accuracy",
       "value": 0.6985668053629218,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.05230666363357367,
      "improvement_points": null,
      "runs": [
       "math:glide-v1.2_en",
       "math:glide-v1.2_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.050277777777777775,
      "gpu_cost_usd": 0.12519166666666667
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "celeris-1.1-decision",
      "name": "celeris-1.1-decision",
      "display_name": "celeris-1.1-decision",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Celeris-1.1 Decision",
      "maker": "Celeris",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 62.320850670365225,
      "raw": {
       "metric": "accuracy",
       "value": 0.6985668053629218,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.04499768839574669,
      "improvement_points": null,
      "runs": [
       "math:celeris-1.1_en",
       "math:celeris-1.1_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "liquid-d1.2-ultra",
      "name": "d1.2-ultra",
      "display_name": "d1.2-ultra",
      "tag": "",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "Liquid d1.2-ultra",
      "maker": "Liquid AI",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 62.320850670365225,
      "raw": {
       "metric": "accuracy",
       "value": 0.6985668053629218,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.04578252722217225,
      "improvement_points": null,
      "runs": [
       "math:d1.2-ultra_en",
       "math:d1.2-ultra_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "perplexity-decider-27b-v1.2",
      "name": "Perplexity Decider 27B v1.2",
      "display_name": "Perplexity Decider 27B v1.2",
      "tag": "API",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Perplexity Decider 27B v1.2 (API)",
      "maker": "Perplexity",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": null,
      "score_0_100": 62.20527045769764,
      "raw": {
       "metric": "accuracy",
       "value": 0.6976421636615812,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.03229675297431705,
      "improvement_points": null,
      "runs": [
       "math:decider-v1.2_en",
       "math:decider-v1.2_en_dontknow"
      ],
      "notes": "hosted API (the maker's service); other setups of this model under its row, not in this entry: Perplexity Decider 27B v1.2 (NVIDIA RTX PRO 6000 Blackwell)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "perplexity-decider-27b-v1.2-with-web",
      "name": "Perplexity Decider 27B v1.2",
      "display_name": "Perplexity Decider 27B v1.2",
      "tag": "API · with web",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Perplexity Decider 27B v1.2 (API, with web)",
      "maker": "Perplexity",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": null,
      "score_0_100": 61.627369394359675,
      "raw": {
       "metric": "accuracy",
       "value": 0.6930189551548774,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.03698135910311147,
      "improvement_points": null,
      "runs": [
       "math:decider-v1.2-web_en",
       "math:decider-v1.2-web_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "turion-ai/turion-jev-31b",
      "commit": "866c0fc6e040cba23fcc72113a80e5c99d5ebab1",
      "model_key": "turion-jev-31b",
      "name": "Turion Jev 31B",
      "display_name": "Turion Jev 31B",
      "tag": "",
      "class_label": "Open · 31B",
      "base_model": null,
      "setup_label": "Turion Jev 31B",
      "maker": "Turion.AI",
      "kind": "decision",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "FP8",
      "score_0_100": 61.280628756356904,
      "raw": {
       "metric": "accuracy",
       "value": 0.6902450300508552,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.046954380490060094,
      "improvement_points": null,
      "runs": [
       "math:turion-jev-31b_en",
       "math:turion-jev-31b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.1775,
      "gpu_cost_usd": 0.037775164130024005
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "subq-snap-1.1",
      "name": "SubQ Snap 1.1",
      "display_name": "SubQ Snap 1.1",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "SubQ Snap 1.1",
      "maker": "Subquadratic",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 60.70272769301896,
      "raw": {
       "metric": "accuracy",
       "value": 0.6856218215441516,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.095208335208,
      "improvement_points": null,
      "runs": [
       "math:subq-1.1_en",
       "math:subq-1.1_en_dontknow"
      ],
      "notes": "hosted API (the maker's service); shown as one row with SubQ Snap Preview (the same answer to every question); every number from SubQ Snap 1.1",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "PelaAI/KnowLine-12B-Gen1",
      "commit": "f26fbc2d64092bb2f8caaedbec243172bb3e908f",
      "model_key": "knowline-12b-gen1",
      "name": "KnowLine-12B-Gen1",
      "display_name": "KnowLine-12B-Gen1",
      "tag": "",
      "class_label": "Open · 12B",
      "base_model": "Gemma 4 12B",
      "setup_label": "KnowLine-12B-Gen1",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 12.0,
      "params_active": 12.0,
      "quantization": "BF16",
      "score_0_100": 57.813222376329165,
      "raw": {
       "metric": "accuracy",
       "value": 0.6625057790106333,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.0397398558622103,
      "improvement_points": null,
      "runs": [
       "math:knowline-12b-gen1_en",
       "math:knowline-12b-gen1_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.03138888888888889,
      "gpu_cost_usd": 0.07815833333333333
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "liquid-d1.2",
      "name": "d1.2",
      "display_name": "d1.2",
      "tag": "",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "Liquid d1.2",
      "maker": "Liquid AI",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 56.88858067498844,
      "raw": {
       "metric": "accuracy",
       "value": 0.6551086453999075,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.0322345924099891,
      "improvement_points": null,
      "runs": [
       "math:d1.2_en",
       "math:d1.2_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "liquid-d1.1",
      "name": "d1.1",
      "display_name": "d1.1",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Liquid d1.1",
      "maker": "Liquid AI",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 56.48404993065187,
      "raw": {
       "metric": "accuracy",
       "value": 0.651872399445215,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.021643242997207185,
      "improvement_points": null,
      "runs": [
       "math:d1.1_en",
       "math:d1.1_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "jev-1.14.0",
      "name": "Jev 1.14",
      "display_name": "Jev 1.14",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Jev 1.14.0",
      "maker": "TypeSafe",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 56.42625982431807,
      "raw": {
       "metric": "accuracy",
       "value": 0.6514100785945446,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.028169209431345345,
      "improvement_points": null,
      "runs": [
       "math:jev-1.14_en",
       "math:jev-1.14_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "jev-1.13.0",
      "name": "Jev 1.13",
      "display_name": "Jev 1.13",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Jev 1.13.0",
      "maker": "TypeSafe",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 55.09708737864077,
      "raw": {
       "metric": "accuracy",
       "value": 0.6407766990291263,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.025681923254738812,
      "improvement_points": null,
      "runs": [
       "math:jev-1.13_en",
       "math:jev-1.13_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "newsbang/adjudex-9b-v5",
      "commit": "ffdcb8254d1b22e40b2bbbf569f143e0b357aeca",
      "model_key": "adjudex-9b-v5",
      "name": "Adjudex 9B v5",
      "display_name": "Adjudex 9B v5",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": "Qwen3.5-9B",
      "setup_label": "Adjudex 9B v5",
      "maker": "newsbang",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 52.612112806287556,
      "raw": {
       "metric": "accuracy",
       "value": 0.6208969024503005,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.04793390671264879,
      "improvement_points": null,
      "runs": [
       "math:adjudex-9b_en",
       "math:adjudex-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.023055555555555555,
      "gpu_cost_usd": 0.05740833333333334
     },
     {
      "repo": "baselabs/elephant-1",
      "commit": "b3f8a93462fb1051a9de031e94a7ff380e254072",
      "model_key": "elephant-1",
      "name": "Elephant-1",
      "display_name": "Elephant-1",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": null,
      "setup_label": "Elephant-1",
      "maker": "Base Labs",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 51.68747110494684,
      "raw": {
       "metric": "accuracy",
       "value": 0.6134997688395747,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.042894290799815056,
      "improvement_points": null,
      "runs": [
       "math:elephant-1_en",
       "math:elephant-1_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.050555555555555555,
      "gpu_cost_usd": 0.12588333333333335
     },
     {
      "repo": "PelaAI/KnowLine-9B-Gen2",
      "commit": "8ec8eb609bc10aa6050ae3754aee419d458fae42",
      "model_key": "knowline-9b-gen2",
      "name": "KnowLine-9B-Gen2",
      "display_name": "KnowLine-9B-Gen2",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": "Qwen3.5-9B",
      "setup_label": "KnowLine-9B-Gen2",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 50.12713823393435,
      "raw": {
       "metric": "accuracy",
       "value": 0.6010171058714748,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.047920111962225155,
      "improvement_points": null,
      "runs": [
       "math:knowline-9b-gen2_en",
       "math:knowline-9b-gen2_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.026944444444444444,
      "gpu_cost_usd": 0.06709166666666667
     },
     {
      "repo": "tattrongvu/Jeffan-9B-1211",
      "commit": "862d1a7a8b64a8d708ddb0b0cd775a740f01841a",
      "model_key": "jeffan-9b-1211",
      "name": "Jeffan-9B-1211",
      "display_name": "Jeffan-9B-1211",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": "Qwen3.5-9B",
      "setup_label": "Jeffan-9B-1211",
      "maker": "tattrongvu",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 49.780397595931575,
      "raw": {
       "metric": "accuracy",
       "value": 0.5982431807674526,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.037523772652044284,
      "improvement_points": null,
      "runs": [
       "math:jeffan-9b_en",
       "math:jeffan-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.025,
      "gpu_cost_usd": 0.06225000000000001
     },
     {
      "repo": "PelaAI/KnowLine-9B-Gen1",
      "commit": "325378d128e9f5eb5088196668f5874c5682b6c0",
      "model_key": "knowline-9b-gen1",
      "name": "KnowLine-9B-Gen1",
      "display_name": "KnowLine-9B-Gen1",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": "Qwen3.5-9B",
      "setup_label": "KnowLine-9B-Gen1",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 49.72260748959777,
      "raw": {
       "metric": "accuracy",
       "value": 0.5977808599167822,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.030535655418292937,
      "improvement_points": null,
      "runs": [
       "math:knowline-9b_en",
       "math:knowline-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.023055555555555555,
      "gpu_cost_usd": 0.05740833333333334
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "microsoft-decision-1",
      "name": "Microsoft-Decision-1",
      "display_name": "Microsoft-Decision-1",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": "Qwen3.5-9B",
      "setup_label": "Microsoft-Decision-1",
      "maker": "Microsoft",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 49.20249653259361,
      "raw": {
       "metric": "accuracy",
       "value": 0.5936199722607489,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.040862520669752544,
      "improvement_points": null,
      "runs": [
       "math:ms-decision-1_en",
       "math:ms-decision-1_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "drex-2.0",
      "name": "Drex 2.0",
      "display_name": "Drex 2.0",
      "tag": "",
      "class_label": "Open · 35B MoE · 3.0B active",
      "base_model": null,
      "setup_label": "Drex 2.0",
      "maker": "Nace.AI",
      "kind": "decision",
      "params_total": 35.0,
      "params_active": 3.0,
      "quantization": null,
      "score_0_100": 48.740175681923255,
      "raw": {
       "metric": "accuracy",
       "value": 0.5899214054553861,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.01879144706426259,
      "improvement_points": null,
      "runs": [
       "math:drex-2.0_en",
       "math:drex-2.0_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "Patrick-18/Dexy-Decision-9B-V2",
      "commit": "4f434e2b5fd9f0addfe3233489571c2dc8ce7a19",
      "model_key": "dexy-decision-9b-v2",
      "name": "Dexy-Decision-9B-V2",
      "display_name": "Dexy-Decision-9B-V2",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": "vllm-sr/d3-flash",
      "setup_label": "Dexy-Decision-9B-V2",
      "maker": "Patrick-18",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 48.509015256588064,
      "raw": {
       "metric": "accuracy",
       "value": 0.5880721220527045,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.12131842805076583,
      "improvement_points": null,
      "runs": [
       "math:dexy-9b_en",
       "math:dexy-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.027777777777777776,
      "gpu_cost_usd": 0.06916666666666667
     },
     {
      "repo": "startlux-models/StartLux-Decision-v1.1-9B-Instant",
      "commit": "18a17f734966b213b2f00fa9c915e8b9af2a9b20",
      "model_key": "startlux-decision-v1.1-9b",
      "name": "StartLux-Decision-v1.1-9B-Instant",
      "display_name": "StartLux-Decision-v1.1-9B-Instant",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": null,
      "setup_label": "StartLux-Decision v1.1 9B",
      "maker": "StartLux",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 48.046694405917705,
      "raw": {
       "metric": "accuracy",
       "value": 0.5843735552473417,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.02693892628266125,
      "improvement_points": null,
      "runs": [
       "math:startlux-9b_en",
       "math:startlux-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.08472222222222223,
      "gpu_cost_usd": 0.018030399154393303
     },
     {
      "repo": "fastino/GLiDE-9B",
      "commit": "ab54df5acba0eebf7df45560ffdaacd64e32df3b",
      "model_key": "glide-9b",
      "name": "GLiDE 9B",
      "display_name": "GLiDE 9B",
      "tag": "",
      "class_label": "Open · 7.9B",
      "base_model": null,
      "setup_label": "GLiDE 9B (open)",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": 7.94,
      "params_active": 7.94,
      "quantization": "BF16",
      "score_0_100": 47.41100323624595,
      "raw": {
       "metric": "accuracy",
       "value": 0.5792880258899676,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.070737570513874,
      "improvement_points": null,
      "runs": [
       "math:glide-9b_en",
       "math:glide-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.10666666666666667,
      "gpu_cost_usd": 0.022700568115695176
     },
     {
      "repo": "scar-ai/vera-medium-omni",
      "commit": "1f23a6cf5304e8e53b7ded53980765d5f8f4047c",
      "model_key": "vera-medium-omni",
      "name": "vera-medium-omni",
      "display_name": "vera-medium-omni",
      "tag": "",
      "class_label": "Open · 4.2B",
      "base_model": "Qwen3.5-4B",
      "setup_label": "vera-medium-omni",
      "maker": "scar-ai",
      "kind": "decision",
      "params_total": 4.2,
      "params_active": 4.2,
      "quantization": "BF16",
      "score_0_100": 46.024040684234855,
      "raw": {
       "metric": "accuracy",
       "value": 0.5681923254738789,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.029700739713361058,
      "improvement_points": null,
      "runs": [
       "math:vera-medium-omni_en",
       "math:vera-medium-omni_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.020277777777777777,
      "gpu_cost_usd": 0.05049166666666667
     },
     {
      "repo": "apertixai/cerno-9b-v1.2",
      "commit": "f02a632fbe3a4698cd1ff5c0b929ea240e2b8eda",
      "model_key": "cerno-9b-v1.2",
      "name": "Cerno 9B v1.2",
      "display_name": "Cerno 9B v1.2",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": null,
      "setup_label": "Cerno 9B v1.2",
      "maker": "Apertix AI",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 45.041608876560325,
      "raw": {
       "metric": "accuracy",
       "value": 0.5603328710124826,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.15733795654184,
      "improvement_points": null,
      "runs": [
       "math:cerno-9b_en",
       "math:cerno-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.09666666666666666,
      "gpu_cost_usd": 0.020572389854848754
     },
     {
      "repo": "PelaAI/KnowLine-4B-Gen5.5",
      "commit": "779944d2cc8ffa19cef386aee83c83bcb967ade2",
      "model_key": "knowline-4b-gen5.5",
      "name": "KnowLine 4B Gen5.5",
      "display_name": "KnowLine 4B Gen5.5",
      "tag": "",
      "class_label": "Open · 4.0B",
      "base_model": null,
      "setup_label": "KnowLine 4B Gen5.5",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "FP8",
      "score_0_100": 44.983818770226534,
      "raw": {
       "metric": "accuracy",
       "value": 0.5598705501618123,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.08632431691710336,
      "improvement_points": null,
      "runs": [
       "math:knowline-4b_en",
       "math:knowline-4b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.04638888888888889,
      "gpu_cost_usd": 0.00987238248781535
     },
     {
      "repo": "perplexity-ai/pplx-decider-flash-v1.3",
      "commit": "d06fb3f2d97e0a52325a0d463c450cf0338f44fe",
      "model_key": "perplexity-decider-flash-v1.3",
      "name": "Perplexity Decider Flash v1.3",
      "display_name": "Perplexity Decider Flash v1.3",
      "tag": "",
      "class_label": "Open · 4.0B",
      "base_model": null,
      "setup_label": "Perplexity Decider Flash v1.3",
      "maker": "Perplexity",
      "kind": "decision",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "BF16",
      "score_0_100": 44.926028663892744,
      "raw": {
       "metric": "accuracy",
       "value": 0.559408229311142,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.1167804804730328,
      "improvement_points": null,
      "runs": [
       "math:decider-flash-v1.3_en",
       "math:decider-flash-v1.3_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.019444444444444445,
      "gpu_cost_usd": 0.04841666666666667
     },
     {
      "repo": "startlux-models/StartLux-Decision-v1.1-4B-Instant",
      "commit": "416088f77cfce09633906a3675c7cd2e5dabcf10",
      "model_key": "startlux-decision-v1.1-4b",
      "name": "StartLux-Decision-v1.1-4B-Instant",
      "display_name": "StartLux-Decision-v1.1-4B-Instant",
      "tag": "",
      "class_label": "Open · 4.0B",
      "base_model": null,
      "setup_label": "StartLux-Decision v1.1 4B",
      "maker": "StartLux",
      "kind": "decision",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "BF16",
      "score_0_100": 44.81044845122515,
      "raw": {
       "metric": "accuracy",
       "value": 0.5584835876098012,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.06650363539344888,
      "improvement_points": null,
      "runs": [
       "math:startlux-4b_en",
       "math:startlux-4b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.041666666666666664,
      "gpu_cost_usd": 0.008867409420193428
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "liquid-d1",
      "name": "d1",
      "display_name": "d1",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Liquid d1",
      "maker": "Liquid AI",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 44.52149791955616,
      "raw": {
       "metric": "accuracy",
       "value": 0.5561719833564494,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.032904607617458656,
      "improvement_points": null,
      "runs": [
       "math:d1_en",
       "math:d1_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "Valen-Team/Valen-Decision-1.5-9B",
      "commit": "5a512117ebc0f2b69f311becfcf47cdef8abf568",
      "model_key": "valen-decision-1.5-9b",
      "name": "Valen-Decision-1.5-9B",
      "display_name": "Valen-Decision-1.5-9B",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": "Qwen3.5-9B",
      "setup_label": "Valen-Decision-1.5-9B",
      "maker": "Valen-Team",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 44.17475728155339,
      "raw": {
       "metric": "accuracy",
       "value": 0.5533980582524272,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.040623399081564475,
      "improvement_points": null,
      "runs": [
       "math:valen-9b_en",
       "math:valen-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.035555555555555556,
      "gpu_cost_usd": 0.08853333333333334
     },
     {
      "repo": "perplexity-ai/pplx-decider-flash-v1.2",
      "commit": "6d771ad7b5ecff5e93b8fc7b90d4c5ccc341954d",
      "model_key": "perplexity-decider-flash-v1.2",
      "name": "Perplexity Decider Flash v1.2",
      "display_name": "Perplexity Decider Flash v1.2",
      "tag": "",
      "class_label": "Open · 4.0B",
      "base_model": null,
      "setup_label": "Perplexity Decider Flash v1.2",
      "maker": "Perplexity",
      "kind": "decision",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "BF16",
      "score_0_100": 44.05917706888581,
      "raw": {
       "metric": "accuracy",
       "value": 0.5524734165510865,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.10375862150665573,
      "improvement_points": null,
      "runs": [
       "math:decider-flash-v1.2_en",
       "math:decider-flash-v1.2_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.026111111111111113,
      "gpu_cost_usd": 0.06501666666666668
     },
     {
      "repo": "ZefanCai/Open-Jev-27B-v1.1",
      "commit": "28cf73067d5b337860bbef3c85b8b82ba8730956",
      "model_key": "open-jev-27b-v1.1",
      "name": "Open-Jev-27B-v1.1",
      "display_name": "Open-Jev-27B-v1.1",
      "tag": "",
      "class_label": "Open · 27B",
      "base_model": "Qwen3.8-27B + adapter",
      "setup_label": "Open-Jev 27B v1.1",
      "maker": "Zefan Cai",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 43.192325473878874,
      "raw": {
       "metric": "accuracy",
       "value": 0.545538603791031,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.21052697404071077,
      "improvement_points": null,
      "runs": [
       "math:open-jev_en",
       "math:open-jev_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); the repository holds an adapter and head on Qwen3.8-27B",
      "gpu_hours": 0.14694444444444443,
      "gpu_cost_usd": 0.36589166666666667
     },
     {
      "repo": "OmniJev/OneJev-27B",
      "commit": "a5e5f55bdbdc9deda44f3fcb5b163776b9b369e8",
      "model_key": "onejev-27b",
      "name": "OneJev-27B",
      "display_name": "OneJev-27B",
      "tag": "",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "OneJev 27B",
      "maker": "OmniJev",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 42.209893666204344,
      "raw": {
       "metric": "accuracy",
       "value": 0.5376791493296348,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.03582799722607488,
      "improvement_points": null,
      "runs": [
       "math:onejev_en",
       "math:onejev_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.05277777777777778,
      "gpu_cost_usd": 0.13141666666666668
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "inception-mercury-decide",
      "name": "Mercury Decide",
      "display_name": "Mercury Decide",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Inception Mercury Decide",
      "maker": "Inception",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 41.86315302820157,
      "raw": {
       "metric": "accuracy",
       "value": 0.5349052242256126,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.15801202201424802,
      "improvement_points": null,
      "runs": [
       "math:mercury-decide_en",
       "math:mercury-decide_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "victorbennett/Verdin-4B-v1.1",
      "commit": "45830d216bb7e45f43f0a482c131a7cafa6d189f",
      "model_key": "verdin-4b-v1.1",
      "name": "Verdin-4B-v1.1",
      "display_name": "Verdin-4B-v1.1",
      "tag": "",
      "class_label": "Open · 4.0B",
      "base_model": "Qwen3.5-4B",
      "setup_label": "Verdin-4B-v1.1",
      "maker": "victorbennett",
      "kind": "decision",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "BF16",
      "score_0_100": 39.37817845584835,
      "raw": {
       "metric": "accuracy",
       "value": 0.5150254276467868,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.056776592633318794,
      "improvement_points": null,
      "runs": [
       "math:verdin-4b_en",
       "math:verdin-4b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.029166666666666667,
      "gpu_cost_usd": 0.07262500000000001
     },
     {
      "repo": "Cloudflare/clef",
      "commit": "2f3de3dd85f379784083b0814d997ab627200f0c",
      "model_key": "clef",
      "name": "Clef",
      "display_name": "Clef",
      "tag": "",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Clef",
      "maker": "Cloudflare",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 38.85806749884419,
      "raw": {
       "metric": "accuracy",
       "value": 0.5108645399907535,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.10962413314840498,
      "improvement_points": null,
      "runs": [
       "math:clef_en",
       "math:clef_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.11416666666666667,
      "gpu_cost_usd": 0.28427500000000006
     },
     {
      "repo": "quincyqiang/chinese-decision-pointer-4b",
      "commit": "889f23e2655e5cba1208bb17f90526092813f5f2",
      "model_key": "chinese-decision-pointer-4b",
      "name": "Chinese Decision Pointer 4B",
      "display_name": "Chinese Decision Pointer 4B",
      "tag": "",
      "class_label": "Open · 4.0B",
      "base_model": "Qwen3.5-4B",
      "setup_label": "Chinese Decision Pointer 4B",
      "maker": "quincyqiang",
      "kind": "decision",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "BF16",
      "score_0_100": 38.04900601017106,
      "raw": {
       "metric": "accuracy",
       "value": 0.5043920480813685,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.1374209486331622,
      "improvement_points": null,
      "runs": [
       "math:chinese-pointer-4b_en",
       "math:chinese-pointer-4b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.01972222222222222,
      "gpu_cost_usd": 0.04910833333333334
     },
     {
      "repo": "Cloudflare/clef-flash",
      "commit": "fde727a287004204b7518dcc983fe64379776712",
      "model_key": "clef-flash",
      "name": "Clef-Flash",
      "display_name": "Clef-Flash",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": null,
      "setup_label": "Clef-flash",
      "maker": "Cloudflare",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 37.23994452149791,
      "raw": {
       "metric": "accuracy",
       "value": 0.49791955617198336,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.026981692094313482,
      "improvement_points": null,
      "runs": [
       "math:clef-flash_en",
       "math:clef-flash_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.0625,
      "gpu_cost_usd": 0.155625
     },
     {
      "repo": "EldanRing/Winnow-12B",
      "commit": "b2b14213dfa252e6d6b543c8b334762e51772d29",
      "model_key": "winnow-12b",
      "name": "Winnow-12B",
      "display_name": "Winnow-12B",
      "tag": "",
      "class_label": "Open · 12B",
      "base_model": null,
      "setup_label": "Winnow-12B",
      "maker": "EldanRing",
      "kind": "decision",
      "params_total": 12.0,
      "params_active": 12.0,
      "quantization": "Q8_0",
      "score_0_100": 33.599167822468786,
      "raw": {
       "metric": "accuracy",
       "value": 0.46879334257975036,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.0839036235732786,
      "improvement_points": null,
      "runs": [
       "math:winnow_en",
       "math:winnow_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); the Q8_0 GGUF, served by winnow-inference at d4631fb",
      "gpu_hours": 0.03333333333333333,
      "gpu_cost_usd": 0.083
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "cygnet",
      "name": "Cygnet",
      "display_name": "Cygnet",
      "tag": "",
      "class_label": "Open · 12B",
      "base_model": "Gemma 4 12B IT",
      "setup_label": "Cygnet",
      "maker": "Blockbrain Labs",
      "kind": "decision",
      "params_total": 11.95,
      "params_active": 11.95,
      "quantization": "BF16",
      "score_0_100": 30.42071197411003,
      "raw": {
       "metric": "accuracy",
       "value": 0.44336569579288027,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.059426522048276566,
      "improvement_points": null,
      "runs": [
       "math:cygnet_en",
       "math:cygnet_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); no weights of its own: google/gemma-4-12B-it at 707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7, unchanged, behind cygnet-recipe's shim (github.com/blockbrain-ai/cygnet-recipe 3cf591c, temperature 3.4); repo left empty so it is not matched with Gemma 4 12B",
      "gpu_hours": 0.02388888888888889,
      "gpu_cost_usd": 0.05948333333333334
     },
     {
      "repo": "wayfind/metask-jev-4b-policy-mix",
      "commit": "ea20fe85b28733b1522721dec119bec50947a869",
      "model_key": "metask-jev-4b",
      "name": "Metask-Jev-4B",
      "display_name": "Metask-Jev-4B",
      "tag": "",
      "class_label": "Open · 4.5B",
      "base_model": null,
      "setup_label": "Metask-Jev 4B",
      "maker": "Metask Lab",
      "kind": "decision",
      "params_total": 4.54,
      "params_active": 4.54,
      "quantization": "BF16",
      "score_0_100": 23.948220064724918,
      "raw": {
       "metric": "accuracy",
       "value": 0.39158576051779936,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.03300643423283703,
      "improvement_points": null,
      "runs": [
       "math:metask_en",
       "math:metask_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.07222222222222222,
      "gpu_cost_usd": 0.17983333333333335
     },
     {
      "repo": "startlux-models/StartLux-Decision-v1.1-2B-Instant",
      "commit": "f1e6cd147dda82c9ffa3e8e725d5c1f175566fb1",
      "model_key": "startlux-decision-v1.1-2b",
      "name": "StartLux-Decision-v1.1-2B-Instant",
      "display_name": "StartLux-Decision-v1.1-2B-Instant",
      "tag": "",
      "class_label": "Open · 2.0B",
      "base_model": null,
      "setup_label": "StartLux-Decision v1.1 2B",
      "maker": "StartLux",
      "kind": "decision",
      "params_total": 2.0,
      "params_active": 2.0,
      "quantization": "BF16",
      "score_0_100": 23.312528895053163,
      "raw": {
       "metric": "accuracy",
       "value": 0.3865002311604253,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.026080989749775452,
      "improvement_points": null,
      "runs": [
       "math:startlux-2b_en",
       "math:startlux-2b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.023055555555555555,
      "gpu_cost_usd": 0.00490663321250703
     },
     {
      "repo": "LiquidAI/d1-2B",
      "commit": "10e7671d44e4b37700020b1eed3f594c5f2af0b4",
      "model_key": "liquid-d1-2b",
      "name": "d1-2B",
      "display_name": "d1-2B",
      "tag": "",
      "class_label": "Open · 1.9B",
      "base_model": "LFM3-VL-2B",
      "setup_label": "Liquid d1-2B (open weights)",
      "maker": "Liquid AI",
      "kind": "decision",
      "params_total": 1.92,
      "params_active": 1.92,
      "quantization": "BF16",
      "score_0_100": 21.347665279704113,
      "raw": {
       "metric": "accuracy",
       "value": 0.37078132223763294,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.05560965380005984,
      "improvement_points": null,
      "runs": [
       "math:d1-2b_en",
       "math:d1-2b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.012222222222222223,
      "gpu_cost_usd": 0.030433333333333333
     },
     {
      "repo": "fastino/GLiDE-2B",
      "commit": "00f122c120cdeed82284106cb77d9bab5a50ad16",
      "model_key": "glide-2b",
      "name": "GLiDE 2B",
      "display_name": "GLiDE 2B",
      "tag": "",
      "class_label": "Open · 1.9B",
      "base_model": null,
      "setup_label": "GLiDE 2B (open)",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": 1.88,
      "params_active": 1.88,
      "quantization": "BF16",
      "score_0_100": 20.943134535367545,
      "raw": {
       "metric": "accuracy",
       "value": 0.36754507628294036,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.17155643235372448,
      "improvement_points": null,
      "runs": [
       "math:glide-2b_en",
       "math:glide-2b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.018055555555555554,
      "gpu_cost_usd": 0.0038425440820838187
     },
     {
      "repo": "PelaAI/KnowLine-2B-Gen1",
      "commit": "53245d470a4ce316e32fd0d18c24e142c697f40b",
      "model_key": "knowline-2b-gen1",
      "name": "KnowLine-2B-Gen1",
      "display_name": "KnowLine-2B-Gen1",
      "tag": "",
      "class_label": "Open · 2.0B",
      "base_model": "Qwen3.5-2B",
      "setup_label": "KnowLine-2B-Gen1",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 2.0,
      "params_active": 2.0,
      "quantization": "BF16",
      "score_0_100": 19.09385113268608,
      "raw": {
       "metric": "accuracy",
       "value": 0.35275080906148865,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.09408551986577202,
      "improvement_points": null,
      "runs": [
       "math:knowline-2b_en",
       "math:knowline-2b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.0325,
      "gpu_cost_usd": 0.006916579347750875
     },
     {
      "repo": "fastino/GLiDE-0.8B",
      "commit": "9eacdeec79fa48ec52a8d6f54b331376b4bbfaf0",
      "model_key": "glide-0.8b",
      "name": "GLiDE 0.8B",
      "display_name": "GLiDE 0.8B",
      "tag": "",
      "class_label": "Open · 0.8B",
      "base_model": null,
      "setup_label": "GLiDE 0.8B (open)",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": 0.75,
      "params_active": 0.75,
      "quantization": "BF16",
      "score_0_100": 15.10633379565418,
      "raw": {
       "metric": "accuracy",
       "value": 0.32085067036523346,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.18775447761695988,
      "improvement_points": null,
      "runs": [
       "math:glide-0.8b_en",
       "math:glide-0.8b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.013055555555555556,
      "gpu_cost_usd": 0.0027784549516606076
     },
     {
      "repo": "startlux-models/StartLux-Decision-v1.1-0.8B-Instant",
      "commit": "6496ae641cc71b149ce9dad69bd97f2a44e7bf11",
      "model_key": "startlux-decision-v1.1-0.8b-instant",
      "name": "StartLux-Decision-v1.1-0.8B-Instant",
      "display_name": "StartLux-Decision-v1.1-0.8B-Instant",
      "tag": "",
      "class_label": "Open · 0.8B",
      "base_model": "Qwen3.5-0.8B",
      "setup_label": "StartLux-Decision-v1.1-0.8B-Instant",
      "maker": "StartLux",
      "kind": "decision",
      "params_total": 0.8,
      "params_active": 0.8,
      "quantization": "BF16",
      "score_0_100": 12.101248266296807,
      "raw": {
       "metric": "accuracy",
       "value": 0.29680998613037446,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.04765301858447206,
      "improvement_points": null,
      "runs": [
       "math:startlux-0.8b_en",
       "math:startlux-0.8b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.015555555555555555,
      "gpu_cost_usd": 0.003310499516872213
     },
     {
      "repo": "StrandsAgents/strands-decider-2B-hobson-v21",
      "commit": "2b52a6235c1b8306bbfa30b00b9d4b74b63a39f5",
      "model_key": "strands-decider-2b",
      "name": "Strands Decider 2B",
      "display_name": "Strands Decider 2B",
      "tag": "",
      "class_label": "Open · 2.0B",
      "base_model": null,
      "setup_label": "Strands Decider 2B",
      "maker": "Strands Agents",
      "kind": "decision",
      "params_total": 2.0,
      "params_active": 2.0,
      "quantization": "BF16",
      "score_0_100": 10.829865926953307,
      "raw": {
       "metric": "accuracy",
       "value": 0.28663892741562647,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.0876370319001387,
      "improvement_points": null,
      "runs": [
       "math:strands_en",
       "math:strands_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); the repository holds a LoRA and head on Qwen3.5-2B-Base",
      "gpu_hours": 0.02277777777777778,
      "gpu_cost_usd": 0.056716666666666665
     },
     {
      "repo": "OmniJev/OneJev1.6-0.8B",
      "commit": "bcd88a382f314b26cb7b1b7f369c1ed3502ee7a0",
      "model_key": "onejev1.6-0.8b",
      "name": "OneJev1.6-0.8B",
      "display_name": "OneJev1.6-0.8B",
      "tag": "",
      "class_label": "Open · 0.8B",
      "base_model": null,
      "setup_label": "OneJev1.6-0.8B",
      "maker": "OmniJev",
      "kind": "decision",
      "params_total": 0.8,
      "params_active": 0.8,
      "quantization": "BF16",
      "score_0_100": 10.309754969949147,
      "raw": {
       "metric": "accuracy",
       "value": 0.2824780397595932,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.09989208737864079,
      "improvement_points": null,
      "runs": [
       "math:onejev1.6-0.8b_en",
       "math:onejev1.6-0.8b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.011111111111111112,
      "gpu_cost_usd": 0.002364642512051581
     },
     {
      "repo": "PelaAI/KnowLine-0.8B-Gen1",
      "commit": "6b27831f7b783c03278bd946d4d8f4f7a98a3798",
      "model_key": "knowline-0.8b-gen1",
      "name": "KnowLine-0.8B-Gen1",
      "display_name": "KnowLine-0.8B-Gen1",
      "tag": "",
      "class_label": "Open · 0.8B",
      "base_model": "Qwen3.5-0.8B",
      "setup_label": "KnowLine-0.8B-Gen1",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 0.8,
      "params_active": 0.8,
      "quantization": "BF16",
      "score_0_100": 10.251964863615349,
      "raw": {
       "metric": "accuracy",
       "value": 0.2820157189089228,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.10540986823469145,
      "improvement_points": null,
      "runs": [
       "math:knowline-0.8b_en",
       "math:knowline-0.8b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.017222222222222222,
      "gpu_cost_usd": 0.003665195893679951
     },
     {
      "repo": "knowledgator/gliclass-v4-edge",
      "commit": "885af1af8a07edf4ed34df83ca0d995ed8201283",
      "model_key": "gliclass-v4-edge",
      "name": "GLiClass v4 Edge",
      "display_name": "GLiClass v4 Edge",
      "tag": "",
      "class_label": "Open · 0.4B",
      "base_model": "DeBERTa-v3-large",
      "setup_label": "GLiClass v4 Edge",
      "maker": null,
      "kind": "decision",
      "params_total": 0.4,
      "params_active": 0.4,
      "quantization": "BF16",
      "score_0_100": 3.3171521035598692,
      "raw": {
       "metric": "accuracy",
       "value": 0.22653721682847897,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.07014710098192448,
      "improvement_points": null,
      "runs": [
       "math:gliclass-v4-edge_en",
       "math:gliclass-v4-edge_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.008611111111111111,
      "gpu_cost_usd": 0.0018325979468399754
     },
     {
      "repo": "fastino/GLiDE-0.4B",
      "commit": "6bda7ab91aeb36dcad6d27828a8871e548c7fc71",
      "model_key": "glide-0.4b",
      "name": "GLiDE 0.4B",
      "display_name": "GLiDE 0.4B",
      "tag": "",
      "class_label": "Open · 0.4B",
      "base_model": null,
      "setup_label": "GLiDE 0.4B (open)",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": 0.4,
      "params_active": 0.4,
      "quantization": "BF16",
      "score_0_100": 0.716597318539066,
      "raw": {
       "metric": "accuracy",
       "value": 0.20573277854831254,
       "questions": 2163,
       "options": 5
      },
      "calibration_error": 0.12157482885587954,
      "improvement_points": null,
      "runs": [
       "math:glide-0.4b_en",
       "math:glide-0.4b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.0077777777777777776,
      "gpu_cost_usd": 0.0016552497584361066
     }
    ]
   },
   "sha256": "df54b81601a42a7d369107d650ac797724c8c33165a649caff50433ee19ea11c"
  },
  "logic-math": {
   "url": "https://zekiworks.github.io/analysis/logic/bench-scores.json",
   "published": true,
   "seen": "2026-11-16",
   "file": {
    "bench": "logic-math",
    "title": "Logic & Math",
    "url": "https://zekiworks.github.io/analysis/logic/",
    "updated": "2026-11-16T21:03:26Z",
    "score": {
     "zero": "chance: 1 of 5 options right on every question (accuracy 20%)",
     "hundred": "every one of the 3,288 Logic and Math questions right (accuracy 100%)"
    },
    "models": [
     {
      "repo": null,
      "commit": null,
      "model_key": "gpt-6-astra",
      "name": "GPT-6 Astra",
      "display_name": "GPT-6 Astra",
      "tag": "chat",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "GPT-6 Astra",
      "maker": "OpenAI",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 98.28923357664232,
      "raw": {
       "metric": "accuracy",
       "value": 0.9863138686131386,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:gpt-6-astra-sub_en",
       "math:gpt-6-astra-sub_en",
       "logic:gpt-6-astra-sub_en_dontknow",
       "math:gpt-6-astra-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "gemini-4-carbon",
      "name": "Gemini 4 Carbon",
      "display_name": "Gemini 4 Carbon",
      "tag": "chat",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "Gemini 4 Carbon (Antigravity subscription)",
      "maker": "Google",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 98.25121654501217,
      "raw": {
       "metric": "accuracy",
       "value": 0.9860097323600974,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:gemini-4-carbon-sub_en",
       "math:gemini-4-carbon-sub_en",
       "logic:gemini-4-carbon-sub_en_dontknow",
       "math:gemini-4-carbon-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "gpt-6.1-sol",
      "name": "GPT-6.1 Sol",
      "display_name": "GPT-6.1 Sol",
      "tag": "chat",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "GPT-6.1 Sol",
      "maker": "OpenAI",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 98.213199513382,
      "raw": {
       "metric": "accuracy",
       "value": 0.985705596107056,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:gpt-6.1-sol-sub_en",
       "math:gpt-6.1-sol-sub_en",
       "logic:gpt-6.1-sol-sub_en_dontknow",
       "math:gpt-6.1-sol-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "qwen4-max",
      "name": "Qwen4-Max",
      "display_name": "Qwen4-Max",
      "tag": "high · chat",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "Qwen4-Max",
      "maker": "Alibaba",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 98.213199513382,
      "raw": {
       "metric": "accuracy",
       "value": 0.985705596107056,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:qwen4-max_en",
       "math:qwen4-max_en",
       "logic:qwen4-max_en_dontknow",
       "math:qwen4-max_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "claude-opus-5.5",
      "name": "Claude Opus 5.5",
      "display_name": "Claude Opus 5.5",
      "tag": "chat",
      "class_label": "Closed · Flagship",
      "base_model": null,
      "setup_label": "Claude Opus 5.5",
      "maker": "Anthropic",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 98.17518248175183,
      "raw": {
       "metric": "accuracy",
       "value": 0.9854014598540146,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:claude-opus-5.5-sub_en",
       "math:claude-opus-5.5-sub_en",
       "logic:claude-opus-5.5-sub_en_dontknow",
       "math:claude-opus-5.5-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "deepseek-v4.1-flash-1017",
      "name": "DeepSeek-V4.1-Flash",
      "display_name": "DeepSeek-V4.1-Flash",
      "tag": "high · chat · 1017",
      "class_label": "Open · 552B MoE · 16B active",
      "base_model": null,
      "setup_label": "DeepSeek-V4.1-Flash 1017",
      "maker": "DeepSeek",
      "kind": "chat",
      "params_total": 552.0,
      "params_active": 16.0,
      "quantization": null,
      "score_0_100": 98.0611313868613,
      "raw": {
       "metric": "accuracy",
       "value": 0.9844890510948905,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:deepseek-v4.1-flash-1017_en",
       "math:deepseek-v4.1-flash-1017_en",
       "logic:deepseek-v4.1-flash-1017_en_dontknow",
       "math:deepseek-v4.1-flash-1017_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "gemini-4-hydrogen",
      "name": "Gemini 4 Hydrogen",
      "display_name": "Gemini 4 Hydrogen",
      "tag": "chat",
      "class_label": "",
      "base_model": null,
      "setup_label": "Gemini 4 Hydrogen (Antigravity subscription)",
      "maker": "Google",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 97.98509732360097,
      "raw": {
       "metric": "accuracy",
       "value": 0.9838807785888077,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:gemini-4-hydrogen-sub_en",
       "math:gemini-4-hydrogen-sub_en",
       "logic:gemini-4-hydrogen-sub_en_dontknow",
       "math:gemini-4-hydrogen-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "claude-sonnet-5.5",
      "name": "Claude Sonnet 5.5",
      "display_name": "Claude Sonnet 5.5",
      "tag": "chat",
      "class_label": "Closed · Workhorse",
      "base_model": null,
      "setup_label": "Claude Sonnet 5.5",
      "maker": "Anthropic",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 97.79501216545012,
      "raw": {
       "metric": "accuracy",
       "value": 0.982360097323601,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:claude-sonnet-5.5-sub_en",
       "math:claude-sonnet-5.5-sub_en",
       "logic:claude-sonnet-5.5-sub_en_dontknow",
       "math:claude-sonnet-5.5-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "inkling-2-small",
      "name": "Inkling 2 Small",
      "display_name": "Inkling 2 Small",
      "tag": "high · chat",
      "class_label": "Open · 594B MoE · 17B active",
      "base_model": null,
      "setup_label": "Inkling 2 Small",
      "maker": "Thinking Machines",
      "kind": "chat",
      "params_total": 594.0,
      "params_active": 17.0,
      "quantization": null,
      "score_0_100": 97.75699513381994,
      "raw": {
       "metric": "accuracy",
       "value": 0.9820559610705596,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:inkling-2-small_en",
       "math:inkling-2-small_en",
       "logic:inkling-2-small_en_dontknow",
       "math:inkling-2-small_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "claude-fable-5.1",
      "name": "Claude Fable 5.1",
      "display_name": "Claude Fable 5.1",
      "tag": "chat",
      "class_label": "Closed · Flagship",
      "base_model": null,
      "setup_label": "Claude Fable 5.1",
      "maker": "Anthropic",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 97.71897810218977,
      "raw": {
       "metric": "accuracy",
       "value": 0.9817518248175182,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:claude-fable-5.1-sub_en",
       "math:claude-fable-5.1-sub_en",
       "logic:claude-fable-5.1-sub_en_dontknow",
       "math:claude-fable-5.1-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "inkling-small",
      "name": "Inkling-Small",
      "display_name": "Inkling-Small",
      "tag": "high · chat · Jul 2026",
      "class_label": "Open · 276B MoE · 12B active",
      "base_model": null,
      "setup_label": "Inkling Small",
      "maker": "Thinking Machines",
      "kind": "chat",
      "params_total": 276.0,
      "params_active": 12.0,
      "quantization": null,
      "score_0_100": 97.71897810218977,
      "raw": {
       "metric": "accuracy",
       "value": 0.9817518248175182,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:inkling-small-64k_en",
       "math:inkling-small-64k_en",
       "logic:inkling-small-64k_en_dontknow",
       "math:inkling-small-64k_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "muse-ember",
      "name": "Muse Ember",
      "display_name": "Muse Ember",
      "tag": "max · chat",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Muse Ember",
      "maker": "Meta",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 97.71897810218977,
      "raw": {
       "metric": "accuracy",
       "value": 0.9817518248175182,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:muse-ember_en",
       "math:muse-ember_en",
       "logic:muse-ember_en_dontknow",
       "math:muse-ember_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "mimo-v2.6-flash",
      "name": "MiMo-V2.6-Flash",
      "display_name": "MiMo-V2.6-Flash",
      "tag": "on · chat",
      "class_label": "Open · 309B MoE · 15B active",
      "base_model": null,
      "setup_label": "MiMo V2.6 Flash",
      "maker": "Xiaomi",
      "kind": "chat",
      "params_total": 309.0,
      "params_active": 15.0,
      "quantization": null,
      "score_0_100": 97.45285888077859,
      "raw": {
       "metric": "accuracy",
       "value": 0.9796228710462287,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:mimo-v2.6-flash-64k_en",
       "math:mimo-v2.6-flash-64k_en",
       "logic:mimo-v2.6-flash-64k_en_dontknow",
       "math:mimo-v2.6-flash-64k_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "qwen3.8-flash",
      "name": "Qwen3.8 Flash",
      "display_name": "Qwen3.8 Flash",
      "tag": "off · chat",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "Qwen3.8 Flash",
      "maker": "Alibaba",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 96.99665450121655,
      "raw": {
       "metric": "accuracy",
       "value": 0.9759732360097324,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:qwen3.8-flash_en",
       "math:qwen3.8-flash_en",
       "logic:qwen3.8-flash_en_dontknow",
       "math:qwen3.8-flash_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "Qwen/Qwen3.8-27B",
      "commit": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
      "model_key": "qwen3.8-27b",
      "name": "Qwen3.8-27B",
      "display_name": "Qwen3.8-27B",
      "tag": "low · chat",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Qwen3.8-27B (chat, reasoning low, on our GPU)",
      "maker": "Alibaba",
      "kind": "chat",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 96.80656934306569,
      "raw": {
       "metric": "accuracy",
       "value": 0.9744525547445255,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:qwen3.8-27b-bf16_en_low",
       "math:qwen3.8-27b-bf16_en_low",
       "logic:qwen3.8-27b-bf16_en_low_dontknow",
       "math:qwen3.8-27b-bf16_en_low_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 2.6669444444444443,
      "gpu_cost_usd": 6.640691666666667
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "gpt-6.1-terra",
      "name": "GPT-6.1 Terra",
      "display_name": "GPT-6.1 Terra",
      "tag": "chat",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "GPT-6.1 Terra",
      "maker": "OpenAI",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 96.76855231143551,
      "raw": {
       "metric": "accuracy",
       "value": 0.9741484184914841,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:gpt-6.1-terra-sub_en",
       "math:gpt-6.1-terra-sub_en",
       "logic:gpt-6.1-terra-sub_en_dontknow",
       "math:gpt-6.1-terra-sub_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "google/gemma-4-31B-it",
      "commit": "842da3794eaa0b77d5f08bae87a17459d91ff475",
      "model_key": "gemma-4-31b",
      "name": "Gemma 4 31B",
      "display_name": "Gemma 4 31B",
      "tag": "chat",
      "class_label": "Open · 31B",
      "base_model": null,
      "setup_label": "Gemma 4 31B (chat, on our GPU)",
      "maker": "Google",
      "kind": "chat",
      "params_total": 30.7,
      "params_active": 30.7,
      "quantization": "FP8",
      "score_0_100": 96.65450121654501,
      "raw": {
       "metric": "accuracy",
       "value": 0.9732360097323601,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:gemma-4-31b-it_en_t0",
       "math:gemma-4-31b-it_en_t0",
       "logic:gemma-4-31b-it_en_t0_dontknow",
       "math:gemma-4-31b-it_en_t0_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); BF16 weights quantized to FP8 by vLLM at load",
      "gpu_hours": 1.0230555555555556,
      "gpu_cost_usd": 2.5474083333333333
     },
     {
      "repo": "Qwen/Qwen3.5-4B",
      "commit": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",
      "model_key": "qwen3.5-4b",
      "name": "Qwen3.5-4B",
      "display_name": "Qwen3.5-4B",
      "tag": "on · chat",
      "class_label": "Open · 4.7B",
      "base_model": null,
      "setup_label": "Qwen3.5-4B (chat, thinking, on our GPU)",
      "maker": "Alibaba",
      "kind": "chat",
      "params_total": 4.66,
      "params_active": 4.66,
      "quantization": "BF16",
      "score_0_100": 96.27433090024331,
      "raw": {
       "metric": "accuracy",
       "value": 0.9701946472019465,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:qwen3.5-4b_en",
       "math:qwen3.5-4b_en",
       "logic:qwen3.5-4b_en_dontknow",
       "math:qwen3.5-4b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: Qwen3.5-4B (off · chat)",
      "gpu_hours": 2.6966666666666668,
      "gpu_cost_usd": 6.714700000000001
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "glm-5.3-flash",
      "name": "GLM-5.3-Flash",
      "display_name": "GLM-5.3-Flash",
      "tag": "high · chat",
      "class_label": "Open · 320B MoE · 18B active",
      "base_model": null,
      "setup_label": "GLM-5.3-Flash",
      "maker": "Z.ai",
      "kind": "chat",
      "params_total": 320.0,
      "params_active": 18.0,
      "quantization": null,
      "score_0_100": 96.08424574209245,
      "raw": {
       "metric": "accuracy",
       "value": 0.9686739659367397,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:glm-5.3-flash_en",
       "math:glm-5.3-flash_en",
       "logic:glm-5.3-flash_en_dontknow",
       "math:glm-5.3-flash_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "minimax-m3.1-flash",
      "name": "MiniMax-M3.1-Flash",
      "display_name": "MiniMax-M3.1-Flash",
      "tag": "high · chat",
      "class_label": "Open · 428B MoE · 23B active",
      "base_model": null,
      "setup_label": "MiniMax-M3.1-Flash",
      "maker": "MiniMax",
      "kind": "chat",
      "params_total": 428.0,
      "params_active": 23.0,
      "quantization": null,
      "score_0_100": 95.97019464720194,
      "raw": {
       "metric": "accuracy",
       "value": 0.9677615571776156,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:minimax-m3.1-flash_en",
       "math:minimax-m3.1-flash_en",
       "logic:minimax-m3.1-flash_en_dontknow",
       "math:minimax-m3.1-flash_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "ibm-granite/granite-4.2-3b",
      "commit": "4e9c275e232f10211ec88644d106a1f6a1b60e06",
      "model_key": "granite-4.2-3b",
      "name": "Granite-4.2-3B",
      "display_name": "Granite-4.2-3B",
      "tag": "high · chat",
      "class_label": "Open · 3.0B",
      "base_model": "Granite-4.1-3B-Base",
      "setup_label": "Granite-4.2-3B (chat, thinking, on our GPU)",
      "maker": "IBM",
      "kind": "chat",
      "params_total": 3.0,
      "params_active": 3.0,
      "quantization": "BF16",
      "score_0_100": 95.7801094890511,
      "raw": {
       "metric": "accuracy",
       "value": 0.9662408759124088,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:granite-4.2-3b-64k_en",
       "math:granite-4.2-3b-64k_en",
       "logic:granite-4.2-3b-64k_en_dontknow",
       "math:granite-4.2-3b-64k_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 5.599722222222223,
      "gpu_cost_usd": 13.943308333333334
     },
     {
      "repo": "openbmb/MiniCPM5-2B",
      "commit": "42433e90dad95123ab87729fd7011e7110629556",
      "model_key": "minicpm5-2b",
      "name": "MiniCPM5-2B",
      "display_name": "MiniCPM5-2B",
      "tag": "on · chat",
      "class_label": "Open · 2.5B",
      "base_model": null,
      "setup_label": "MiniCPM5-2B (chat, thinking, on our GPU)",
      "maker": "OpenBMB",
      "kind": "chat",
      "params_total": 2.52,
      "params_active": 2.52,
      "quantization": "BF16",
      "score_0_100": 94.60158150851579,
      "raw": {
       "metric": "accuracy",
       "value": 0.9568126520681265,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:minicpm5-2b_en",
       "math:minicpm5-2b_en",
       "logic:minicpm5-2b_en_dontknow",
       "math:minicpm5-2b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: MiniCPM5-2B (off · chat); accuracy is the mean of 3 seed runs (min 0.9544, max 0.9595): seed 0 minicpm5-2b_en+minicpm5-2b_en 3145/3288, seed 1 minicpm5-2b-seed1_en+minicpm5-2b-seed1_en 3138/3288, seed 2 minicpm5-2b-seed2_en+minicpm5-2b-seed2_en 3155/3288",
      "gpu_hours": 1.8980555555555556,
      "gpu_cost_usd": 4.726158333333334
     },
     {
      "repo": "google/gemma-4-12B-it",
      "commit": "43c9d014b231474fa7212dbb48a762566c2b255d",
      "model_key": "gemma-4-12b",
      "name": "Gemma 4 12B Unified",
      "display_name": "Gemma 4 12B Unified",
      "tag": "chat",
      "class_label": "Open · 12B",
      "base_model": null,
      "setup_label": "Gemma 4 12B (chat, on our GPU)",
      "maker": "Google",
      "kind": "chat",
      "params_total": 11.95,
      "params_active": 11.95,
      "quantization": "BF16",
      "score_0_100": 93.80322384428224,
      "raw": {
       "metric": "accuracy",
       "value": 0.9504257907542579,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:gemma-4-12b-it_en",
       "math:gemma-4-12b-it_en",
       "logic:gemma-4-12b-it_en_dontknow",
       "math:gemma-4-12b-it_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.9894444444444443,
      "gpu_cost_usd": 2.463716666666667
     },
     {
      "repo": "LiquidAI/LFM2.5-24B-A2B",
      "commit": "1769516ac588c4c558089c94f27d96b0dc67b183",
      "model_key": "lfm2.5-24b-a2b",
      "name": "LFM2.5-24B-A2B",
      "display_name": "LFM2.5-24B-A2B",
      "tag": "on · chat",
      "class_label": "Open · 24B MoE · 2.5B active",
      "base_model": null,
      "setup_label": "LFM2.5-24B-A2B (chat, thinking, on our GPU)",
      "maker": "Liquid AI",
      "kind": "chat",
      "params_total": 24.0,
      "params_active": 2.45,
      "quantization": "BF16",
      "score_0_100": 93.49908759124087,
      "raw": {
       "metric": "accuracy",
       "value": 0.947992700729927,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:lfm2.5-24b-a2b_en",
       "math:lfm2.5-24b-a2b_en",
       "logic:lfm2.5-24b-a2b_en_dontknow",
       "math:lfm2.5-24b-a2b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 4.232222222222222,
      "gpu_cost_usd": 10.538233333333334
     },
     {
      "repo": "LiquidAI/LFM3-1.7B",
      "commit": "0dc4268b561499c648c7ba7c9466bf3d8d01517a",
      "model_key": "lfm3-1.7b",
      "name": "LFM3-1.7B",
      "display_name": "LFM3-1.7B",
      "tag": "off · chat",
      "class_label": "Open · 1.7B",
      "base_model": null,
      "setup_label": "LFM3-1.7B (chat, thinking off, on our GPU)",
      "maker": null,
      "kind": "chat",
      "params_total": 1.73,
      "params_active": 1.73,
      "quantization": "BF16",
      "score_0_100": 92.73874695863745,
      "raw": {
       "metric": "accuracy",
       "value": 0.9419099756690997,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:lfm3-1.7b-vllm-nothink_en",
       "math:lfm3-1.7b-vllm-nothink_en",
       "logic:lfm3-1.7b-vllm-nothink_en_dontknow",
       "math:lfm3-1.7b-vllm-nothink_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10); other setups of this model under its row, not in this entry: LFM3-1.7B (on · chat), LFM3-1.7B (off · chat · NVIDIA RTX PRO 6000 Blackwell)",
      "gpu_hours": 3.1822222222222223,
      "gpu_cost_usd": 0.6772336154515728
     },
     {
      "repo": "google/gemma-4-E4B-it",
      "commit": "4ecc2ba63bcac23ba4a767fc87025b1450f84c9c",
      "model_key": "gemma-4-e4b",
      "name": "Gemma 4 E4B",
      "display_name": "Gemma 4 E4B",
      "tag": "chat",
      "class_label": "Open · 8.0B",
      "base_model": null,
      "setup_label": "Gemma 4 E4B (chat, on our GPU)",
      "maker": "Google",
      "kind": "chat",
      "params_total": 8.0,
      "params_active": 8.0,
      "quantization": "BF16",
      "score_0_100": 91.25608272506082,
      "raw": {
       "metric": "accuracy",
       "value": 0.9300486618004866,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:gemma-4-e4b-it_en",
       "math:gemma-4-e4b-it_en",
       "logic:gemma-4-e4b-it_en_dontknow",
       "math:gemma-4-e4b-it_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: Gemma 4 E4B (on · chat), Gemma 4 E4B (on · chat · 32k cap), Gemma 4 E4B (on · chat · 32k cap · sampled); 4.5B effective parameters",
      "gpu_hours": 0.616111111111111,
      "gpu_cost_usd": 1.5341166666666668
     },
     {
      "repo": "google/gemma-4-E2B-it",
      "commit": "4adf15748de9e1a0097b6274cee9c15836c41c37",
      "model_key": "gemma-4-e2b",
      "name": "Gemma 4 E2B",
      "display_name": "Gemma 4 E2B",
      "tag": "chat",
      "class_label": "Open · 5.1B",
      "base_model": null,
      "setup_label": "Gemma 4 E2B (chat, on our GPU)",
      "maker": "Google",
      "kind": "chat",
      "params_total": 5.1,
      "params_active": 5.1,
      "quantization": "BF16",
      "score_0_100": 85.74361313868614,
      "raw": {
       "metric": "accuracy",
       "value": 0.8859489051094891,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:gemma-4-e2b-it_en",
       "math:gemma-4-e2b-it_en",
       "logic:gemma-4-e2b-it_en_dontknow",
       "math:gemma-4-e2b-it_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: Gemma 4 E2B (on · chat), Gemma 4 E2B (on · chat · 32k cap), Gemma 4 E2B (on · chat · 32k cap · sampled); 2.3B effective parameters",
      "gpu_hours": 0.4127777777777778,
      "gpu_cost_usd": 1.0278166666666668
     },
     {
      "repo": "LiquidAI/LFM2.5-2.6B",
      "commit": "7141b56cd4650af95c7c7b86983443aed050c96a",
      "model_key": "lfm2.5-2.6b",
      "name": "LFM2.5-2.6B",
      "display_name": "LFM2.5-2.6B",
      "tag": "on · chat",
      "class_label": "Open · 2.7B",
      "base_model": null,
      "setup_label": "LFM2.5-2.6B (chat, thinking, on our GPU)",
      "maker": "Liquid AI",
      "kind": "chat",
      "params_total": 2.69,
      "params_active": 2.69,
      "quantization": "BF16",
      "score_0_100": 84.52706812652069,
      "raw": {
       "metric": "accuracy",
       "value": 0.8762165450121655,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:lfm2.5-2.6b_en",
       "math:lfm2.5-2.6b_en",
       "logic:lfm2.5-2.6b_en_dontknow",
       "math:lfm2.5-2.6b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 1.3808333333333334,
      "gpu_cost_usd": 3.438275
     },
     {
      "repo": "Qwen/Qwen3-14B",
      "commit": "40c069824f4251a91eefaf281ebe4c544efd3e18",
      "model_key": "qwen3-14b",
      "name": "Qwen3-14B",
      "display_name": "Qwen3-14B",
      "tag": "off · chat",
      "class_label": "Open · 15B",
      "base_model": null,
      "setup_label": "Qwen3-14B (chat, reasoning off, on our GPU)",
      "maker": "Alibaba",
      "kind": "chat",
      "params_total": 14.8,
      "params_active": 14.8,
      "quantization": "BF16",
      "score_0_100": 81.71380778588806,
      "raw": {
       "metric": "accuracy",
       "value": 0.8537104622871047,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:qwen3-14b_en",
       "math:qwen3-14b_en",
       "logic:qwen3-14b_en_dontknow",
       "math:qwen3-14b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 1.108888888888889,
      "gpu_cost_usd": 2.7611333333333334
     },
     {
      "repo": "Qwen/Qwen3.5-2B",
      "commit": "15852e8c16360a2fea060d615a32b45270f8a8fc",
      "model_key": "qwen3.5-2b",
      "name": "Qwen3.5-2B",
      "display_name": "Qwen3.5-2B",
      "tag": "off · chat",
      "class_label": "Open · 2.0B",
      "base_model": null,
      "setup_label": "Qwen3.5-2B (chat, thinking off, on our GPU)",
      "maker": "Alibaba",
      "kind": "chat",
      "params_total": 2.0,
      "params_active": 2.0,
      "quantization": "BF16",
      "score_0_100": 80.5352798053528,
      "raw": {
       "metric": "accuracy",
       "value": 0.8442822384428224,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:qwen3.5-2b-nothink_en",
       "math:qwen3.5-2b-nothink_en",
       "logic:qwen3.5-2b-nothink_en_dontknow",
       "math:qwen3.5-2b-nothink_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: Qwen3.5-2B (on · chat)",
      "gpu_hours": 1.0405555555555555,
      "gpu_cost_usd": 2.590983333333334
     },
     {
      "repo": "LiquidAI/LFM2.5-8B-A1B",
      "commit": "dd83f5cb02aa2938b66bba9a9998d38d0517c1f3",
      "model_key": "lfm2.5-8b-a1b",
      "name": "LFM2.5-8B-A1B",
      "display_name": "LFM2.5-8B-A1B",
      "tag": "on · chat",
      "class_label": "Open · 8.3B MoE · 1.5B active",
      "base_model": null,
      "setup_label": "LFM2.5-8B-A1B (chat, thinking, on our GPU)",
      "maker": "Liquid AI",
      "kind": "chat",
      "params_total": 8.3,
      "params_active": 1.5,
      "quantization": "BF16",
      "score_0_100": 79.8889902676399,
      "raw": {
       "metric": "accuracy",
       "value": 0.8391119221411192,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:lfm2.5-8b-a1b_en",
       "math:lfm2.5-8b-a1b_en",
       "logic:lfm2.5-8b-a1b_en_dontknow",
       "math:lfm2.5-8b-a1b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 2.5261111111111108,
      "gpu_cost_usd": 6.290016666666668
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "gpt-6-luna",
      "name": "GPT-6 Luna",
      "display_name": "GPT-6 Luna",
      "tag": "chat",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "GPT-6 Luna (low)",
      "maker": "OpenAI",
      "kind": "chat",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 76.80961070559611,
      "raw": {
       "metric": "accuracy",
       "value": 0.8144768856447688,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:gpt-6-luna-low-sub_en",
       "math:gpt-6-luna-low-sub_en"
      ],
      "notes": "hosted API (the maker's service); no complete 'I don't know' run",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "KRAFTON/Raon-1-nano",
      "commit": "fcf2d1d55e84e2f9fbe7b5d724dcfc4d861cfcee",
      "model_key": "raon-1-nano",
      "name": "Raon-1 nano",
      "display_name": "Raon-1 nano",
      "tag": "off · chat",
      "class_label": "Open · 3.1B",
      "base_model": "Raon-1 mini preview (MoE-to-Dense pruning and distillation)",
      "setup_label": "Raon-1 nano (chat, thinking off, KRAFTON vLLM fork, on our GPU)",
      "maker": "KRAFTON",
      "kind": "chat",
      "params_total": 3.1,
      "params_active": 3.1,
      "quantization": null,
      "score_0_100": 76.46745742092456,
      "raw": {
       "metric": "accuracy",
       "value": 0.8117396593673966,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:raon-1-nano-nothink_en",
       "math:raon-1-nano-nothink_en"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: Raon-1 nano (on · chat); 1.5B effective parameters; no complete 'I don't know' run",
      "gpu_hours": 5.839444444444444,
      "gpu_cost_usd": 14.540216666666668
     },
     {
      "repo": "ecloudtech/Erk-14B",
      "commit": "8584b14a829d5cc803bc227f44e9012e36e8ad04",
      "model_key": "erk-14b",
      "name": "Erk-14B",
      "display_name": "Erk-14B",
      "tag": "off · chat",
      "class_label": "Open · 15B",
      "base_model": null,
      "setup_label": "Erk-14B (chat, reasoning off, on our GPU)",
      "maker": "eCloud Technology",
      "kind": "chat",
      "params_total": 14.77,
      "params_active": 14.77,
      "quantization": "BF16",
      "score_0_100": 72.81782238442821,
      "raw": {
       "metric": "accuracy",
       "value": 0.7825425790754258,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "logic:erk-14b_en",
       "math:erk-14b_en",
       "logic:erk-14b_en_dontknow",
       "math:erk-14b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.4072222222222222,
      "gpu_cost_usd": 1.0139833333333335
     },
     {
      "repo": "MachinePulseAI/karpo-decision-31B",
      "commit": "d69cdd57eab23271cbe65236ca8a00d80d7bb8c7",
      "model_key": "karpo-decision-31b",
      "name": "Karpo Decision 31B",
      "display_name": "Karpo Decision 31B",
      "tag": "",
      "class_label": "Open · 31B",
      "base_model": null,
      "setup_label": "karpo-decision-31B",
      "maker": "MachinePulse",
      "kind": "decision",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "BF16",
      "score_0_100": 68.02767639902676,
      "raw": {
       "metric": "accuracy",
       "value": 0.7442214111922141,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.027356011778684193,
      "improvement_points": null,
      "runs": [
       "logic:karpo-decision-31b_en",
       "math:karpo-decision-31b_en",
       "logic:karpo-decision-31b_en_dontknow",
       "math:karpo-decision-31b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.15583333333333332,
      "gpu_cost_usd": 0.388025
     },
     {
      "repo": "PelaAI/KnowLine-31B-Gen2",
      "commit": "fd99a9ed2255d8a67dd47a44cc429e638f611593",
      "model_key": "knowline-31b-gen2",
      "name": "KnowLine-31B-Gen2",
      "display_name": "KnowLine-31B-Gen2",
      "tag": "",
      "class_label": "Open · 31B",
      "base_model": "Gemma 4 31B",
      "setup_label": "KnowLine-31B-Gen2",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "BF16",
      "score_0_100": 66.20285888077859,
      "raw": {
       "metric": "accuracy",
       "value": 0.7296228710462287,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.02388174276626831,
      "improvement_points": null,
      "runs": [
       "logic:knowline-31b-gen2_en",
       "math:knowline-31b-gen2_en",
       "logic:knowline-31b-gen2_en_dontknow",
       "math:knowline-31b-gen2_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.12277777777777776,
      "gpu_cost_usd": 0.3057166666666667
     },
     {
      "repo": "decisionmodels/REAX-1.3",
      "commit": "fa4bf7ee3bb56a8bc421dad090631be1c5845c2d",
      "model_key": "reax-1.3",
      "name": "REAX 1.3",
      "display_name": "REAX 1.3",
      "tag": "",
      "class_label": "Open · 57B",
      "base_model": null,
      "setup_label": "REAX 1.3",
      "maker": "REAX",
      "kind": "decision",
      "params_total": 57.36,
      "params_active": 57.36,
      "quantization": "FP8",
      "score_0_100": 64.49209245742094,
      "raw": {
       "metric": "accuracy",
       "value": 0.7159367396593674,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.04659104431713918,
      "improvement_points": null,
      "runs": [
       "logic:reax13_en",
       "math:reax13_en",
       "logic:reax13_en_dontknow",
       "math:reax13_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); weights byte-identical to decisionmodels/REAX-1.3 at this commit; served by the REAX engine (github.com/reaxlabs/reax-engine ff8f595)",
      "gpu_hours": 0.19583333333333333,
      "gpu_cost_usd": 0.4876250000000001
     },
     {
      "repo": "PelaAI/KnowLine-31B-Gen1.5",
      "commit": "8e56aa1b236a6dd1064dcfd22ba1e5e360e4c140",
      "model_key": "knowline-31b-gen1.5",
      "name": "KnowLine-31B-Gen1.5",
      "display_name": "KnowLine-31B-Gen1.5",
      "tag": "",
      "class_label": "Open · 31B",
      "base_model": null,
      "setup_label": "KnowLine 31B Gen1.5",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "BF16",
      "score_0_100": 63.883819951338204,
      "raw": {
       "metric": "accuracy",
       "value": 0.7110705596107056,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.025510875826781288,
      "improvement_points": null,
      "runs": [
       "logic:knowline-31b_en",
       "math:knowline-31b_en",
       "logic:knowline-31b_en_dontknow",
       "math:knowline-31b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.8011111111111111,
      "gpu_cost_usd": 0.170490725118919
     },
     {
      "repo": "Poindexter-Labs/Copernicus-31B",
      "commit": "be715259e97a743341f2d7a3e6a6298dab23ca0e",
      "model_key": "copernicus-31b",
      "name": "Copernicus-31B",
      "display_name": "Copernicus-31B",
      "tag": "",
      "class_label": "Open · 31B",
      "base_model": null,
      "setup_label": "Copernicus-31B",
      "maker": "Poindexter Labs",
      "kind": "decision",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "BF16",
      "score_0_100": 63.69373479318735,
      "raw": {
       "metric": "accuracy",
       "value": 0.7095498783454988,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.050027622239130536,
      "improvement_points": null,
      "runs": [
       "logic:copernicus-31b_en",
       "math:copernicus-31b_en",
       "logic:copernicus-31b_en_dontknow",
       "math:copernicus-31b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.13472222222222222,
      "gpu_cost_usd": 0.33545833333333336
     },
     {
      "repo": "sthanika-ai/Sieve-31B-v3",
      "commit": "8fcee858f7f20a50bfa3c8f984c22adb3cbb46a2",
      "model_key": "sieve-31b-v3",
      "name": "Sieve 31B v3",
      "display_name": "Sieve 31B v3",
      "tag": "",
      "class_label": "Open · 31B",
      "base_model": null,
      "setup_label": "Sieve 31B v3",
      "maker": "Sthānika AI",
      "kind": "decision",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "BF16",
      "score_0_100": 63.16149635036496,
      "raw": {
       "metric": "accuracy",
       "value": 0.7052919708029197,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.025529653284671536,
      "improvement_points": null,
      "runs": [
       "logic:sieve-31b_en",
       "math:sieve-31b_en",
       "logic:sieve-31b_en_dontknow",
       "math:sieve-31b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.7305555555555556,
      "gpu_cost_usd": 0.15547524516739145
     },
     {
      "repo": "logitshq/Logits-Sigma-1-Duo",
      "commit": "a0c1b6e8242916aafdcf9a5a49c1a952cb56e091",
      "model_key": "logits-sigma-1-duo",
      "name": "Logits Sigma-1 Duo",
      "display_name": "Logits Sigma-1 Duo",
      "tag": "",
      "class_label": "Open · 57B",
      "base_model": null,
      "setup_label": "Logits Sigma-1 Duo",
      "maker": "Logits",
      "kind": "decision",
      "params_total": 57.36,
      "params_active": 57.36,
      "quantization": "FP8",
      "score_0_100": 61.56478102189781,
      "raw": {
       "metric": "accuracy",
       "value": 0.6925182481751825,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.059973442131064064,
      "improvement_points": null,
      "runs": [
       "logic:logits-sigma-1-duo_en",
       "math:logits-sigma-1-duo_en",
       "logic:logits-sigma-1-duo_en_dontknow",
       "math:logits-sigma-1-duo_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.2177777777777778,
      "gpu_cost_usd": 0.5422666666666667
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "liquid-d1-ultra",
      "name": "d1-ultra",
      "display_name": "d1-ultra",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Liquid d1-ultra",
      "maker": "Liquid AI",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 59.43582725060828,
      "raw": {
       "metric": "accuracy",
       "value": 0.6754866180048662,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.020692482363286382,
      "improvement_points": null,
      "runs": [
       "logic:d1-ultra_en",
       "math:d1-ultra_en",
       "logic:d1-ultra_en_dontknow",
       "math:d1-ultra_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "fastino/GLiDE-27B",
      "commit": "4d32743ae96b74f0116f050d37742bbd43c04e7d",
      "model_key": "glide-27b",
      "name": "GLiDE 27B",
      "display_name": "GLiDE 27B",
      "tag": "",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "GLiDE 27B (open)",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 59.169708029197075,
      "raw": {
       "metric": "accuracy",
       "value": 0.6733576642335767,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.04139597970976491,
      "improvement_points": null,
      "runs": [
       "logic:glide-27b_en",
       "math:glide-27b_en",
       "logic:glide-27b_en_dontknow",
       "math:glide-27b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.4411111111111111,
      "gpu_cost_usd": 0.09387630772844777
     },
     {
      "repo": "baselabs/mammoth-1",
      "commit": "61cec44c281c693992e3d6ebeb44dd3402dfa186",
      "model_key": "mammoth-1",
      "name": "Mammoth-1",
      "display_name": "Mammoth-1",
      "tag": "",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Mammoth-1",
      "maker": "Base Labs",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 59.13169099756691,
      "raw": {
       "metric": "accuracy",
       "value": 0.6730535279805353,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.0315745736009732,
      "improvement_points": null,
      "runs": [
       "logic:mammoth-1_en",
       "math:mammoth-1_en",
       "logic:mammoth-1_en_dontknow",
       "math:mammoth-1_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.15833333333333333,
      "gpu_cost_usd": 0.39425000000000004
     },
     {
      "repo": "perplexity-ai/pplx-decider-v1.3",
      "commit": "9c2ebb27470d3dec2ad7ab727727e40d142c7213",
      "model_key": "perplexity-decider-27b-v1.3",
      "name": "Perplexity Decider 27B v1.3",
      "display_name": "Perplexity Decider 27B v1.3",
      "tag": "NVIDIA RTX PRO 6000 Blackwell",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Perplexity Decider 27B v1.3 (local GPU)",
      "maker": "Perplexity",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 58.02919708029196,
      "raw": {
       "metric": "accuracy",
       "value": 0.6642335766423357,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.05881688896428786,
      "improvement_points": null,
      "runs": [
       "logic:decider-v1.3-local_en",
       "math:decider-v1.3-local_en",
       "logic:decider-v1.3-local_en_dontknow",
       "math:decider-v1.3-local_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.09388888888888888,
      "gpu_cost_usd": 0.23378333333333334
     },
     {
      "repo": "vllm-sr/d3-ultra",
      "commit": "52ba9e7a00402f9560c7fba8c039608e7ddee7fb",
      "model_key": "vllm-sr-d3-ultra",
      "name": "d3-ultra",
      "display_name": "d3-ultra",
      "tag": "",
      "class_label": "Open · 26B",
      "base_model": null,
      "setup_label": "vLLM-SR d3-ultra",
      "maker": "vLLM Semantic Router",
      "kind": "decision",
      "params_total": 26.09,
      "params_active": 26.09,
      "quantization": "BF16",
      "score_0_100": 57.15480535279804,
      "raw": {
       "metric": "accuracy",
       "value": 0.6572384428223844,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.10115505435596814,
      "improvement_points": null,
      "runs": [
       "logic:d3-ultra_en",
       "math:d3-ultra_en",
       "logic:d3-ultra_en_dontknow",
       "math:d3-ultra_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.12638888888888888,
      "gpu_cost_usd": 0.31470833333333337
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "fastino-glide",
      "name": "GLiDE",
      "display_name": "GLiDE",
      "tag": "API",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Fastino GLiDE",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 56.50851581508515,
      "raw": {
       "metric": "accuracy",
       "value": 0.6520681265206812,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.05343725853741545,
      "improvement_points": null,
      "runs": [
       "logic:glide_en",
       "math:glide_en",
       "logic:glide_en_dontknow",
       "math:glide_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "vllm-sr/d3",
      "commit": "20f37b9a892acc13d4b07a4f3c4f11b2518c615c",
      "model_key": "vllm-sr-d3",
      "name": "d3",
      "display_name": "d3",
      "tag": "",
      "class_label": "Open · 26B",
      "base_model": null,
      "setup_label": "vLLM-SR d3",
      "maker": "vLLM Semantic Router",
      "kind": "decision",
      "params_total": 26.09,
      "params_active": 26.09,
      "quantization": "BF16",
      "score_0_100": 56.20437956204379,
      "raw": {
       "metric": "accuracy",
       "value": 0.6496350364963503,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.04423810997461454,
      "improvement_points": null,
      "runs": [
       "logic:d3_en",
       "math:d3_en",
       "logic:d3_en_dontknow",
       "math:d3_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.06111111111111111,
      "gpu_cost_usd": 0.15216666666666667
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "drex-max-1.1",
      "name": "Drex Max v1.1",
      "display_name": "Drex Max v1.1",
      "tag": "",
      "class_label": "Closed · Efficient (provisional)",
      "base_model": null,
      "setup_label": "Drex Max 1.1",
      "maker": "Nace.AI",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 56.052311435523116,
      "raw": {
       "metric": "accuracy",
       "value": 0.648418491484185,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.05762789601484136,
      "improvement_points": null,
      "runs": [
       "logic:drex-max-1.1_en",
       "math:drex-max-1.1_en",
       "logic:drex-max-1.1_en_dontknow",
       "math:drex-max-1.1_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "fastino/GLiDE",
      "commit": "a7e45896be7c96ba3354891da790c5ec99c30b6c",
      "model_key": "fastino-glide-v1.2",
      "name": "GLiDE",
      "display_name": "GLiDE",
      "tag": "v1.2",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Fastino GLiDE v1.2",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 56.052311435523116,
      "raw": {
       "metric": "accuracy",
       "value": 0.648418491484185,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.05086397570791474,
      "improvement_points": null,
      "runs": [
       "logic:glide-v1.2_en",
       "math:glide-v1.2_en",
       "logic:glide-v1.2_en_dontknow",
       "math:glide-v1.2_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.08722222222222223,
      "gpu_cost_usd": 0.21718333333333334
     },
     {
      "repo": "nace-ai/drex-max-v1",
      "commit": "cac65e1982a0237e996419ba559004522d9a33ac",
      "model_key": "drex-max-v1",
      "name": "Drex Max",
      "display_name": "Drex Max",
      "tag": "",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Drex Max v1",
      "maker": "Nace.AI",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 55.52007299270072,
      "raw": {
       "metric": "accuracy",
       "value": 0.6441605839416058,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.13306835732526895,
      "improvement_points": null,
      "runs": [
       "logic:drex-max-v1_en",
       "math:drex-max-v1_en",
       "logic:drex-max-v1_en_dontknow",
       "math:drex-max-v1_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); served with Nace's drex-decision-models code at e206b27",
      "gpu_hours": 0.08416666666666667,
      "gpu_cost_usd": 0.209575
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "perplexity-decider-27b-v1.2",
      "name": "Perplexity Decider 27B v1.2",
      "display_name": "Perplexity Decider 27B v1.2",
      "tag": "API",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Perplexity Decider 27B v1.2 (API)",
      "maker": "Perplexity",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": null,
      "score_0_100": 55.17791970802919,
      "raw": {
       "metric": "accuracy",
       "value": 0.6414233576642335,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.030739340675629948,
      "improvement_points": null,
      "runs": [
       "logic:decider-v1.2_en",
       "math:decider-v1.2_en",
       "logic:decider-v1.2_en_dontknow",
       "math:decider-v1.2_en_dontknow"
      ],
      "notes": "hosted API (the maker's service); other setups of this model under its row, not in this entry: Perplexity Decider 27B v1.2 (NVIDIA RTX PRO 6000 Blackwell)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "gemini-decision-1",
      "name": "Gemini Decision 1",
      "display_name": "Gemini Decision 1",
      "tag": "20261111",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Gemini Decision 1",
      "maker": "Google",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 55.06386861313869,
      "raw": {
       "metric": "accuracy",
       "value": 0.6405109489051095,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.020467706279056145,
      "improvement_points": null,
      "runs": [
       "logic:gemini-decision-1_en",
       "math:gemini-decision-1_en",
       "logic:gemini-decision-1_en_dontknow",
       "math:gemini-decision-1_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "liquid-d1.2-ultra",
      "name": "d1.2-ultra",
      "display_name": "d1.2-ultra",
      "tag": "",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "Liquid d1.2-ultra",
      "maker": "Liquid AI",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 54.493613138686136,
      "raw": {
       "metric": "accuracy",
       "value": 0.6359489051094891,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.03239794297434979,
      "improvement_points": null,
      "runs": [
       "logic:d1.2-ultra_en",
       "math:d1.2-ultra_en",
       "logic:d1.2-ultra_en_dontknow",
       "math:d1.2-ultra_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "perplexity-decider-27b-v1.2-with-web",
      "name": "Perplexity Decider 27B v1.2",
      "display_name": "Perplexity Decider 27B v1.2",
      "tag": "API · with web",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Perplexity Decider 27B v1.2 (API, with web)",
      "maker": "Perplexity",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": null,
      "score_0_100": 54.45559610705596,
      "raw": {
       "metric": "accuracy",
       "value": 0.6356447688564477,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.036947861603181184,
      "improvement_points": null,
      "runs": [
       "logic:decider-v1.2-web_en",
       "math:decider-v1.2-web_en",
       "logic:decider-v1.2-web_en_dontknow",
       "math:decider-v1.2-web_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "turion-ai/turion-jev-31b",
      "commit": "866c0fc6e040cba23fcc72113a80e5c99d5ebab1",
      "model_key": "turion-jev-31b",
      "name": "Turion Jev 31B",
      "display_name": "Turion Jev 31B",
      "tag": "",
      "class_label": "Open · 31B",
      "base_model": null,
      "setup_label": "Turion Jev 31B",
      "maker": "Turion.AI",
      "kind": "decision",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "FP8",
      "score_0_100": 54.11344282238443,
      "raw": {
       "metric": "accuracy",
       "value": 0.6329075425790754,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.035488999391727505,
      "improvement_points": null,
      "runs": [
       "logic:turion-jev-31b_en",
       "math:turion-jev-31b_en",
       "logic:turion-jev-31b_en_dontknow",
       "math:turion-jev-31b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.30916666666666665,
      "gpu_cost_usd": 0.06579617789783523
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "subq-snap-1.1",
      "name": "SubQ Snap 1.1",
      "display_name": "SubQ Snap 1.1",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "SubQ Snap 1.1",
      "maker": "Subquadratic",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 53.771289537712896,
      "raw": {
       "metric": "accuracy",
       "value": 0.6301703163017032,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.10473201449715741,
      "improvement_points": null,
      "runs": [
       "logic:subq-1.1_en",
       "math:subq-1.1_en",
       "logic:subq-1.1_en_dontknow",
       "math:subq-1.1_en_dontknow"
      ],
      "notes": "hosted API (the maker's service); shown as one row with SubQ Snap Preview (the same answer to every question); every number from SubQ Snap 1.1",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "celeris-1-decision",
      "name": "celeris-1-decision",
      "display_name": "celeris-1-decision",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Celeris-1 Decision",
      "maker": "Celeris",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 52.78284671532846,
      "raw": {
       "metric": "accuracy",
       "value": 0.6222627737226277,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.05759124087591238,
      "improvement_points": null,
      "runs": [
       "logic:celeris_en",
       "math:celeris_en",
       "logic:celeris_en_dontknow",
       "math:celeris_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "celeris-1.1-decision",
      "name": "celeris-1.1-decision",
      "display_name": "celeris-1.1-decision",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Celeris-1.1 Decision",
      "maker": "Celeris",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 52.66879562043795,
      "raw": {
       "metric": "accuracy",
       "value": 0.6213503649635036,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.05964111922141121,
      "improvement_points": null,
      "runs": [
       "logic:celeris-1.1_en",
       "math:celeris-1.1_en",
       "logic:celeris-1.1_en_dontknow",
       "math:celeris-1.1_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "PelaAI/KnowLine-12B-Gen1",
      "commit": "f26fbc2d64092bb2f8caaedbec243172bb3e908f",
      "model_key": "knowline-12b-gen1",
      "name": "KnowLine-12B-Gen1",
      "display_name": "KnowLine-12B-Gen1",
      "tag": "",
      "class_label": "Open · 12B",
      "base_model": "Gemma 4 12B",
      "setup_label": "KnowLine-12B-Gen1",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 12.0,
      "params_active": 12.0,
      "quantization": "BF16",
      "score_0_100": 51.11009732360097,
      "raw": {
       "metric": "accuracy",
       "value": 0.6088807785888077,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.029121863781880038,
      "improvement_points": null,
      "runs": [
       "logic:knowline-12b-gen1_en",
       "math:knowline-12b-gen1_en",
       "logic:knowline-12b-gen1_en_dontknow",
       "math:knowline-12b-gen1_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.05472222222222223,
      "gpu_cost_usd": 0.13625833333333334
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "liquid-d1.2",
      "name": "d1.2",
      "display_name": "d1.2",
      "tag": "",
      "class_label": "Closed · Undisclosed",
      "base_model": null,
      "setup_label": "Liquid d1.2",
      "maker": "Liquid AI",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 49.89355231143551,
      "raw": {
       "metric": "accuracy",
       "value": 0.5991484184914841,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.024843457817886924,
      "improvement_points": null,
      "runs": [
       "logic:d1.2_en",
       "math:d1.2_en",
       "logic:d1.2_en_dontknow",
       "math:d1.2_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "jev-1.14.0",
      "name": "Jev 1.14",
      "display_name": "Jev 1.14",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Jev 1.14.0",
      "maker": "TypeSafe",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 49.17122871046229,
      "raw": {
       "metric": "accuracy",
       "value": 0.5933698296836983,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.017801094890510953,
      "improvement_points": null,
      "runs": [
       "logic:jev-1.14_en",
       "math:jev-1.14_en",
       "logic:jev-1.14_en_dontknow",
       "math:jev-1.14_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "liquid-d1.1",
      "name": "d1.1",
      "display_name": "d1.1",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Liquid d1.1",
      "maker": "Liquid AI",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 48.29683698296836,
      "raw": {
       "metric": "accuracy",
       "value": 0.5863746958637469,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.008077240223353713,
      "improvement_points": null,
      "runs": [
       "logic:d1.1_en",
       "math:d1.1_en",
       "logic:d1.1_en_dontknow",
       "math:d1.1_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "jev-1.13.0",
      "name": "Jev 1.13",
      "display_name": "Jev 1.13",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Jev 1.13.0",
      "maker": "TypeSafe",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 47.49847931873479,
      "raw": {
       "metric": "accuracy",
       "value": 0.5799878345498783,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.0318734793187348,
      "improvement_points": null,
      "runs": [
       "logic:jev-1.13_en",
       "math:jev-1.13_en",
       "logic:jev-1.13_en_dontknow",
       "math:jev-1.13_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "newsbang/adjudex-9b-v5",
      "commit": "ffdcb8254d1b22e40b2bbbf569f143e0b357aeca",
      "model_key": "adjudex-9b-v5",
      "name": "Adjudex 9B v5",
      "display_name": "Adjudex 9B v5",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": "Qwen3.5-9B",
      "setup_label": "Adjudex 9B v5",
      "maker": "newsbang",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 45.90176399026763,
      "raw": {
       "metric": "accuracy",
       "value": 0.5672141119221411,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.05615898820458099,
      "improvement_points": null,
      "runs": [
       "logic:adjudex-9b_en",
       "math:adjudex-9b_en",
       "logic:adjudex-9b_en_dontknow",
       "math:adjudex-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.03722222222222222,
      "gpu_cost_usd": 0.09268333333333334
     },
     {
      "repo": "baselabs/elephant-1",
      "commit": "b3f8a93462fb1051a9de031e94a7ff380e254072",
      "model_key": "elephant-1",
      "name": "Elephant-1",
      "display_name": "Elephant-1",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": null,
      "setup_label": "Elephant-1",
      "maker": "Base Labs",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 45.29349148418491,
      "raw": {
       "metric": "accuracy",
       "value": 0.5623479318734793,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.05632416940389293,
      "improvement_points": null,
      "runs": [
       "logic:elephant-1_en",
       "math:elephant-1_en",
       "logic:elephant-1_en_dontknow",
       "math:elephant-1_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.07694444444444444,
      "gpu_cost_usd": 0.1915916666666667
     },
     {
      "repo": "tattrongvu/Jeffan-9B-1211",
      "commit": "862d1a7a8b64a8d708ddb0b0cd775a740f01841a",
      "model_key": "jeffan-9b-1211",
      "name": "Jeffan-9B-1211",
      "display_name": "Jeffan-9B-1211",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": "Qwen3.5-9B",
      "setup_label": "Jeffan-9B-1211",
      "maker": "tattrongvu",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 44.799270072992705,
      "raw": {
       "metric": "accuracy",
       "value": 0.5583941605839416,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.03609648592290957,
      "improvement_points": null,
      "runs": [
       "logic:jeffan-9b_en",
       "math:jeffan-9b_en",
       "logic:jeffan-9b_en_dontknow",
       "math:jeffan-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.042777777777777776,
      "gpu_cost_usd": 0.10651666666666668
     },
     {
      "repo": "PelaAI/KnowLine-9B-Gen2",
      "commit": "8ec8eb609bc10aa6050ae3754aee419d458fae42",
      "model_key": "knowline-9b-gen2",
      "name": "KnowLine-9B-Gen2",
      "display_name": "KnowLine-9B-Gen2",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": "Qwen3.5-9B",
      "setup_label": "KnowLine-9B-Gen2",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 43.20255474452554,
      "raw": {
       "metric": "accuracy",
       "value": 0.5456204379562044,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.03635415202928956,
      "improvement_points": null,
      "runs": [
       "logic:knowline-9b-gen2_en",
       "math:knowline-9b-gen2_en",
       "logic:knowline-9b-gen2_en_dontknow",
       "math:knowline-9b-gen2_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.045,
      "gpu_cost_usd": 0.11205000000000001
     },
     {
      "repo": "fastino/GLiDE-9B",
      "commit": "ab54df5acba0eebf7df45560ffdaacd64e32df3b",
      "model_key": "glide-9b",
      "name": "GLiDE 9B",
      "display_name": "GLiDE 9B",
      "tag": "",
      "class_label": "Open · 7.9B",
      "base_model": null,
      "setup_label": "GLiDE 9B (open)",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": 7.94,
      "params_active": 7.94,
      "quantization": "BF16",
      "score_0_100": 43.08850364963503,
      "raw": {
       "metric": "accuracy",
       "value": 0.5447080291970803,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.06680273588737132,
      "improvement_points": null,
      "runs": [
       "logic:glide-9b_en",
       "math:glide-9b_en",
       "logic:glide-9b_en_dontknow",
       "math:glide-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.17694444444444446,
      "gpu_cost_usd": 0.037656932004421426
     },
     {
      "repo": "PelaAI/KnowLine-9B-Gen1",
      "commit": "325378d128e9f5eb5088196668f5874c5682b6c0",
      "model_key": "knowline-9b-gen1",
      "name": "KnowLine-9B-Gen1",
      "display_name": "KnowLine-9B-Gen1",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": "Qwen3.5-9B",
      "setup_label": "KnowLine-9B-Gen1",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 43.08850364963503,
      "raw": {
       "metric": "accuracy",
       "value": 0.5447080291970803,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.021630261863242076,
      "improvement_points": null,
      "runs": [
       "logic:knowline-9b_en",
       "math:knowline-9b_en",
       "logic:knowline-9b_en_dontknow",
       "math:knowline-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.03944444444444445,
      "gpu_cost_usd": 0.09821666666666667
     },
     {
      "repo": "startlux-models/StartLux-Decision-v1.1-9B-Instant",
      "commit": "18a17f734966b213b2f00fa9c915e8b9af2a9b20",
      "model_key": "startlux-decision-v1.1-9b",
      "name": "StartLux-Decision-v1.1-9B-Instant",
      "display_name": "StartLux-Decision-v1.1-9B-Instant",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": null,
      "setup_label": "StartLux-Decision v1.1 9B",
      "maker": "StartLux",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 42.974452554744516,
      "raw": {
       "metric": "accuracy",
       "value": 0.5437956204379562,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.02746704204461614,
      "improvement_points": null,
      "runs": [
       "logic:startlux-9b_en",
       "math:startlux-9b_en",
       "logic:startlux-9b_en_dontknow",
       "math:startlux-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.13777777777777778,
      "gpu_cost_usd": 0.0293215671494396
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "microsoft-decision-1",
      "name": "Microsoft-Decision-1",
      "display_name": "Microsoft-Decision-1",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": "Qwen3.5-9B",
      "setup_label": "Microsoft-Decision-1",
      "maker": "Microsoft",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 42.82238442822384,
      "raw": {
       "metric": "accuracy",
       "value": 0.5425790754257908,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.048894690921314585,
      "improvement_points": null,
      "runs": [
       "logic:ms-decision-1_en",
       "math:ms-decision-1_en",
       "logic:ms-decision-1_en_dontknow",
       "math:ms-decision-1_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "Patrick-18/Dexy-Decision-9B-V2",
      "commit": "4f434e2b5fd9f0addfe3233489571c2dc8ce7a19",
      "model_key": "dexy-decision-9b-v2",
      "name": "Dexy-Decision-9B-V2",
      "display_name": "Dexy-Decision-9B-V2",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": "vllm-sr/d3-flash",
      "setup_label": "Dexy-Decision-9B-V2",
      "maker": "Patrick-18",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 42.7463503649635,
      "raw": {
       "metric": "accuracy",
       "value": 0.541970802919708,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.13093113363487935,
      "improvement_points": null,
      "runs": [
       "logic:dexy-9b_en",
       "math:dexy-9b_en",
       "logic:dexy-9b_en_dontknow",
       "math:dexy-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.043055555555555555,
      "gpu_cost_usd": 0.10720833333333334
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "drex-2.0",
      "name": "Drex 2.0",
      "display_name": "Drex 2.0",
      "tag": "",
      "class_label": "Open · 35B MoE · 3.0B active",
      "base_model": null,
      "setup_label": "Drex 2.0",
      "maker": "Nace.AI",
      "kind": "decision",
      "params_total": 35.0,
      "params_active": 3.0,
      "quantization": null,
      "score_0_100": 41.41575425790754,
      "raw": {
       "metric": "accuracy",
       "value": 0.5313260340632603,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.011098965936739638,
      "improvement_points": null,
      "runs": [
       "logic:drex-2.0_en",
       "math:drex-2.0_en",
       "logic:drex-2.0_en_dontknow",
       "math:drex-2.0_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "scar-ai/vera-medium-omni",
      "commit": "1f23a6cf5304e8e53b7ded53980765d5f8f4047c",
      "model_key": "vera-medium-omni",
      "name": "vera-medium-omni",
      "display_name": "vera-medium-omni",
      "tag": "",
      "class_label": "Open · 4.2B",
      "base_model": "Qwen3.5-4B",
      "setup_label": "vera-medium-omni",
      "maker": "scar-ai",
      "kind": "decision",
      "params_total": 4.2,
      "params_active": 4.2,
      "quantization": "BF16",
      "score_0_100": 40.57937956204379,
      "raw": {
       "metric": "accuracy",
       "value": 0.5246350364963503,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.03501024939172748,
      "improvement_points": null,
      "runs": [
       "logic:vera-medium-omni_en",
       "math:vera-medium-omni_en",
       "logic:vera-medium-omni_en_dontknow",
       "math:vera-medium-omni_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.03194444444444444,
      "gpu_cost_usd": 0.07954166666666668
     },
     {
      "repo": "startlux-models/StartLux-Decision-v1.1-4B-Instant",
      "commit": "416088f77cfce09633906a3675c7cd2e5dabcf10",
      "model_key": "startlux-decision-v1.1-4b",
      "name": "StartLux-Decision-v1.1-4B-Instant",
      "display_name": "StartLux-Decision-v1.1-4B-Instant",
      "tag": "",
      "class_label": "Open · 4.0B",
      "base_model": null,
      "setup_label": "StartLux-Decision v1.1 4B",
      "maker": "StartLux",
      "kind": "decision",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "BF16",
      "score_0_100": 40.541362530413615,
      "raw": {
       "metric": "accuracy",
       "value": 0.524330900243309,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.05373157575576887,
      "improvement_points": null,
      "runs": [
       "logic:startlux-4b_en",
       "math:startlux-4b_en",
       "logic:startlux-4b_en_dontknow",
       "math:startlux-4b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.0661111111111111,
      "gpu_cost_usd": 0.014069622946706906
     },
     {
      "repo": "perplexity-ai/pplx-decider-flash-v1.3",
      "commit": "d06fb3f2d97e0a52325a0d463c450cf0338f44fe",
      "model_key": "perplexity-decider-flash-v1.3",
      "name": "Perplexity Decider Flash v1.3",
      "display_name": "Perplexity Decider Flash v1.3",
      "tag": "",
      "class_label": "Open · 4.0B",
      "base_model": null,
      "setup_label": "Perplexity Decider Flash v1.3",
      "maker": "Perplexity",
      "kind": "decision",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "BF16",
      "score_0_100": 40.12317518248175,
      "raw": {
       "metric": "accuracy",
       "value": 0.5209854014598541,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.11131961883276667,
      "improvement_points": null,
      "runs": [
       "logic:decider-flash-v1.3_en",
       "math:decider-flash-v1.3_en",
       "logic:decider-flash-v1.3_en_dontknow",
       "math:decider-flash-v1.3_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.03138888888888889,
      "gpu_cost_usd": 0.07815833333333334
     },
     {
      "repo": "apertixai/cerno-9b-v1.2",
      "commit": "f02a632fbe3a4698cd1ff5c0b929ea240e2b8eda",
      "model_key": "cerno-9b-v1.2",
      "name": "Cerno 9B v1.2",
      "display_name": "Cerno 9B v1.2",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": null,
      "setup_label": "Cerno 9B v1.2",
      "maker": "Apertix AI",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 39.59093673965936,
      "raw": {
       "metric": "accuracy",
       "value": 0.5167274939172749,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.16156213503649636,
      "improvement_points": null,
      "runs": [
       "logic:cerno-9b_en",
       "math:cerno-9b_en",
       "logic:cerno-9b_en_dontknow",
       "math:cerno-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.16722222222222222,
      "gpu_cost_usd": 0.03558786980637629
     },
     {
      "repo": "Valen-Team/Valen-Decision-1.5-9B",
      "commit": "5a512117ebc0f2b69f311becfcf47cdef8abf568",
      "model_key": "valen-decision-1.5-9b",
      "name": "Valen-Decision-1.5-9B",
      "display_name": "Valen-Decision-1.5-9B",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": "Qwen3.5-9B",
      "setup_label": "Valen-Decision-1.5-9B",
      "maker": "Valen-Team",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 39.476885644768856,
      "raw": {
       "metric": "accuracy",
       "value": 0.5158150851581509,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.045180792918975296,
      "improvement_points": null,
      "runs": [
       "logic:valen-9b_en",
       "math:valen-9b_en",
       "logic:valen-9b_en_dontknow",
       "math:valen-9b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.05472222222222222,
      "gpu_cost_usd": 0.13625833333333334
     },
     {
      "repo": "PelaAI/KnowLine-4B-Gen5.5",
      "commit": "779944d2cc8ffa19cef386aee83c83bcb967ade2",
      "model_key": "knowline-4b-gen5.5",
      "name": "KnowLine 4B Gen5.5",
      "display_name": "KnowLine 4B Gen5.5",
      "tag": "",
      "class_label": "Open · 4.0B",
      "base_model": null,
      "setup_label": "KnowLine 4B Gen5.5",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "FP8",
      "score_0_100": 39.134732360097324,
      "raw": {
       "metric": "accuracy",
       "value": 0.5130778588807786,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.10365906116831469,
      "improvement_points": null,
      "runs": [
       "logic:knowline-4b_en",
       "math:knowline-4b_en",
       "logic:knowline-4b_en_dontknow",
       "math:knowline-4b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.0775,
      "gpu_cost_usd": 0.016493381521559776
     },
     {
      "repo": "perplexity-ai/pplx-decider-flash-v1.2",
      "commit": "6d771ad7b5ecff5e93b8fc7b90d4c5ccc341954d",
      "model_key": "perplexity-decider-flash-v1.2",
      "name": "Perplexity Decider Flash v1.2",
      "display_name": "Perplexity Decider Flash v1.2",
      "tag": "",
      "class_label": "Open · 4.0B",
      "base_model": null,
      "setup_label": "Perplexity Decider Flash v1.2",
      "maker": "Perplexity",
      "kind": "decision",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "BF16",
      "score_0_100": 38.86861313868613,
      "raw": {
       "metric": "accuracy",
       "value": 0.5109489051094891,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.08868503250528151,
      "improvement_points": null,
      "runs": [
       "logic:decider-flash-v1.2_en",
       "math:decider-flash-v1.2_en",
       "logic:decider-flash-v1.2_en_dontknow",
       "math:decider-flash-v1.2_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.04166666666666667,
      "gpu_cost_usd": 0.10375000000000001
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "inception-mercury-decide",
      "name": "Mercury Decide",
      "display_name": "Mercury Decide",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Inception Mercury Decide",
      "maker": "Inception",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 38.26034063260341,
      "raw": {
       "metric": "accuracy",
       "value": 0.5060827250608273,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.15653227083997467,
      "improvement_points": null,
      "runs": [
       "logic:mercury-decide_en",
       "math:mercury-decide_en",
       "logic:mercury-decide_en_dontknow",
       "math:mercury-decide_en_dontknow"
      ],
      "notes": "hosted API via OpenRouter",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "liquid-d1",
      "name": "d1",
      "display_name": "d1",
      "tag": "",
      "class_label": "Closed · Efficient",
      "base_model": null,
      "setup_label": "Liquid d1",
      "maker": "Liquid AI",
      "kind": "decision",
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 38.184306569343065,
      "raw": {
       "metric": "accuracy",
       "value": 0.5054744525547445,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.05821117407187681,
      "improvement_points": null,
      "runs": [
       "logic:d1_en",
       "math:d1_en",
       "logic:d1_en_dontknow",
       "math:d1_en_dontknow"
      ],
      "notes": "hosted API (the maker's service)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "repo": "ZefanCai/Open-Jev-27B-v1.1",
      "commit": "28cf73067d5b337860bbef3c85b8b82ba8730956",
      "model_key": "open-jev-27b-v1.1",
      "name": "Open-Jev-27B-v1.1",
      "display_name": "Open-Jev-27B-v1.1",
      "tag": "",
      "class_label": "Open · 27B",
      "base_model": "Qwen3.8-27B + adapter",
      "setup_label": "Open-Jev 27B v1.1",
      "maker": "Zefan Cai",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 38.070255474452544,
      "raw": {
       "metric": "accuracy",
       "value": 0.5045620437956204,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.18263166301686395,
      "improvement_points": null,
      "runs": [
       "logic:open-jev_en",
       "math:open-jev_en",
       "logic:open-jev_en_dontknow",
       "math:open-jev_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); the repository holds an adapter and head on Qwen3.8-27B",
      "gpu_hours": 0.28,
      "gpu_cost_usd": 0.6972
     },
     {
      "repo": "OmniJev/OneJev-27B",
      "commit": "a5e5f55bdbdc9deda44f3fcb5b163776b9b369e8",
      "model_key": "onejev-27b",
      "name": "OneJev-27B",
      "display_name": "OneJev-27B",
      "tag": "",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "OneJev 27B",
      "maker": "OmniJev",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 36.58759124087591,
      "raw": {
       "metric": "accuracy",
       "value": 0.4927007299270073,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.03139984793187349,
      "improvement_points": null,
      "runs": [
       "logic:onejev_en",
       "math:onejev_en",
       "logic:onejev_en_dontknow",
       "math:onejev_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.09166666666666667,
      "gpu_cost_usd": 0.22825
     },
     {
      "repo": "Cloudflare/clef",
      "commit": "2f3de3dd85f379784083b0814d997ab627200f0c",
      "model_key": "clef",
      "name": "Clef",
      "display_name": "Clef",
      "tag": "",
      "class_label": "Open · 27B",
      "base_model": null,
      "setup_label": "Clef",
      "maker": "Cloudflare",
      "kind": "decision",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 34.458637469586364,
      "raw": {
       "metric": "accuracy",
       "value": 0.47566909975669097,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.12585444038929441,
      "improvement_points": null,
      "runs": [
       "logic:clef_en",
       "math:clef_en",
       "logic:clef_en_dontknow",
       "math:clef_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.17777777777777776,
      "gpu_cost_usd": 0.44266666666666676
     },
     {
      "repo": "victorbennett/Verdin-4B-v1.1",
      "commit": "45830d216bb7e45f43f0a482c131a7cafa6d189f",
      "model_key": "verdin-4b-v1.1",
      "name": "Verdin-4B-v1.1",
      "display_name": "Verdin-4B-v1.1",
      "tag": "",
      "class_label": "Open · 4.0B",
      "base_model": "Qwen3.5-4B",
      "setup_label": "Verdin-4B-v1.1",
      "maker": "victorbennett",
      "kind": "decision",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "BF16",
      "score_0_100": 34.34458637469586,
      "raw": {
       "metric": "accuracy",
       "value": 0.4747566909975669,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.07301977921359004,
      "improvement_points": null,
      "runs": [
       "logic:verdin-4b_en",
       "math:verdin-4b_en",
       "logic:verdin-4b_en_dontknow",
       "math:verdin-4b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.04416666666666667,
      "gpu_cost_usd": 0.10997500000000002
     },
     {
      "repo": "quincyqiang/chinese-decision-pointer-4b",
      "commit": "889f23e2655e5cba1208bb17f90526092813f5f2",
      "model_key": "chinese-decision-pointer-4b",
      "name": "Chinese Decision Pointer 4B",
      "display_name": "Chinese Decision Pointer 4B",
      "tag": "",
      "class_label": "Open · 4.0B",
      "base_model": "Qwen3.5-4B",
      "setup_label": "Chinese Decision Pointer 4B",
      "maker": "quincyqiang",
      "kind": "decision",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "BF16",
      "score_0_100": 33.73631386861313,
      "raw": {
       "metric": "accuracy",
       "value": 0.4698905109489051,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.14337543762223412,
      "improvement_points": null,
      "runs": [
       "logic:chinese-pointer-4b_en",
       "math:chinese-pointer-4b_en",
       "logic:chinese-pointer-4b_en_dontknow",
       "math:chinese-pointer-4b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.03305555555555555,
      "gpu_cost_usd": 0.08230833333333334
     },
     {
      "repo": "Cloudflare/clef-flash",
      "commit": "fde727a287004204b7518dcc983fe64379776712",
      "model_key": "clef-flash",
      "name": "Clef-Flash",
      "display_name": "Clef-Flash",
      "tag": "",
      "class_label": "Open · 9.0B",
      "base_model": null,
      "setup_label": "Clef-flash",
      "maker": "Cloudflare",
      "kind": "decision",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 32.59580291970802,
      "raw": {
       "metric": "accuracy",
       "value": 0.4607664233576642,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.05666110097323602,
      "improvement_points": null,
      "runs": [
       "logic:clef-flash_en",
       "math:clef-flash_en",
       "logic:clef-flash_en_dontknow",
       "math:clef-flash_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.09916666666666667,
      "gpu_cost_usd": 0.246925
     },
     {
      "repo": "EldanRing/Winnow-12B",
      "commit": "b2b14213dfa252e6d6b543c8b334762e51772d29",
      "model_key": "winnow-12b",
      "name": "Winnow-12B",
      "display_name": "Winnow-12B",
      "tag": "",
      "class_label": "Open · 12B",
      "base_model": null,
      "setup_label": "Winnow-12B",
      "maker": "EldanRing",
      "kind": "decision",
      "params_total": 12.0,
      "params_active": 12.0,
      "quantization": "Q8_0",
      "score_0_100": 30.542883211678834,
      "raw": {
       "metric": "accuracy",
       "value": 0.4443430656934307,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.10460486276550544,
      "improvement_points": null,
      "runs": [
       "logic:winnow_en",
       "math:winnow_en",
       "logic:winnow_en_dontknow",
       "math:winnow_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); the Q8_0 GGUF, served by winnow-inference at d4631fb",
      "gpu_hours": 0.05666666666666667,
      "gpu_cost_usd": 0.1411
     },
     {
      "repo": null,
      "commit": null,
      "model_key": "cygnet",
      "name": "Cygnet",
      "display_name": "Cygnet",
      "tag": "",
      "class_label": "Open · 12B",
      "base_model": "Gemma 4 12B IT",
      "setup_label": "Cygnet",
      "maker": "Blockbrain Labs",
      "kind": "decision",
      "params_total": 11.95,
      "params_active": 11.95,
      "quantization": "BF16",
      "score_0_100": 28.261861313868607,
      "raw": {
       "metric": "accuracy",
       "value": 0.4260948905109489,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.07390293444681291,
      "improvement_points": null,
      "runs": [
       "logic:cygnet_en",
       "math:cygnet_en",
       "logic:cygnet_en_dontknow",
       "math:cygnet_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); no weights of its own: google/gemma-4-12B-it at 707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7, unchanged, behind cygnet-recipe's shim (github.com/blockbrain-ai/cygnet-recipe 3cf591c, temperature 3.4); repo left empty so it is not matched with Gemma 4 12B",
      "gpu_hours": 0.03777777777777778,
      "gpu_cost_usd": 0.09406666666666667
     },
     {
      "repo": "wayfind/metask-jev-4b-policy-mix",
      "commit": "ea20fe85b28733b1522721dec119bec50947a869",
      "model_key": "metask-jev-4b",
      "name": "Metask-Jev-4B",
      "display_name": "Metask-Jev-4B",
      "tag": "",
      "class_label": "Open · 4.5B",
      "base_model": null,
      "setup_label": "Metask-Jev 4B",
      "maker": "Metask Lab",
      "kind": "decision",
      "params_total": 4.54,
      "params_active": 4.54,
      "quantization": "BF16",
      "score_0_100": 20.92457420924574,
      "raw": {
       "metric": "accuracy",
       "value": 0.36739659367396593,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.02210388753173417,
      "improvement_points": null,
      "runs": [
       "logic:metask_en",
       "math:metask_en",
       "logic:metask_en_dontknow",
       "math:metask_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.11333333333333333,
      "gpu_cost_usd": 0.2822
     },
     {
      "repo": "startlux-models/StartLux-Decision-v1.1-2B-Instant",
      "commit": "f1e6cd147dda82c9ffa3e8e725d5c1f175566fb1",
      "model_key": "startlux-decision-v1.1-2b",
      "name": "StartLux-Decision-v1.1-2B-Instant",
      "display_name": "StartLux-Decision-v1.1-2B-Instant",
      "tag": "",
      "class_label": "Open · 2.0B",
      "base_model": null,
      "setup_label": "StartLux-Decision v1.1 2B",
      "maker": "StartLux",
      "kind": "decision",
      "params_total": 2.0,
      "params_active": 2.0,
      "quantization": "BF16",
      "score_0_100": 19.93613138686131,
      "raw": {
       "metric": "accuracy",
       "value": 0.3594890510948905,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.03276380893703266,
      "improvement_points": null,
      "runs": [
       "logic:startlux-2b_en",
       "math:startlux-2b_en",
       "logic:startlux-2b_en_dontknow",
       "math:startlux-2b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.03638888888888889,
      "gpu_cost_usd": 0.007744204226968927
     },
     {
      "repo": "fastino/GLiDE-2B",
      "commit": "00f122c120cdeed82284106cb77d9bab5a50ad16",
      "model_key": "glide-2b",
      "name": "GLiDE 2B",
      "display_name": "GLiDE 2B",
      "tag": "",
      "class_label": "Open · 1.9B",
      "base_model": null,
      "setup_label": "GLiDE 2B (open)",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": 1.88,
      "params_active": 1.88,
      "quantization": "BF16",
      "score_0_100": 19.02372262773722,
      "raw": {
       "metric": "accuracy",
       "value": 0.3521897810218978,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.1683616191457715,
      "improvement_points": null,
      "runs": [
       "logic:glide-2b_en",
       "math:glide-2b_en",
       "logic:glide-2b_en_dontknow",
       "math:glide-2b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.03,
      "gpu_cost_usd": 0.0063845347825392685
     },
     {
      "repo": "LiquidAI/d1-2B",
      "commit": "10e7671d44e4b37700020b1eed3f594c5f2af0b4",
      "model_key": "liquid-d1-2b",
      "name": "d1-2B",
      "display_name": "d1-2B",
      "tag": "",
      "class_label": "Open · 1.9B",
      "base_model": "LFM3-VL-2B",
      "setup_label": "Liquid d1-2B (open weights)",
      "maker": "Liquid AI",
      "kind": "decision",
      "params_total": 1.92,
      "params_active": 1.92,
      "quantization": "BF16",
      "score_0_100": 18.605535279805352,
      "raw": {
       "metric": "accuracy",
       "value": 0.3488442822384428,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.0592241765431583,
      "improvement_points": null,
      "runs": [
       "logic:d1-2b_en",
       "math:d1-2b_en",
       "logic:d1-2b_en_dontknow",
       "math:d1-2b_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
      "gpu_hours": 0.01888888888888889,
      "gpu_cost_usd": 0.04703333333333333
     },
     {
      "repo": "PelaAI/KnowLine-2B-Gen1",
      "commit": "53245d470a4ce316e32fd0d18c24e142c697f40b",
      "model_key": "knowline-2b-gen1",
      "name": "KnowLine-2B-Gen1",
      "display_name": "KnowLine-2B-Gen1",
      "tag": "",
      "class_label": "Open · 2.0B",
      "base_model": "Qwen3.5-2B",
      "setup_label": "KnowLine-2B-Gen1",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 2.0,
      "params_active": 2.0,
      "quantization": "BF16",
      "score_0_100": 17.579075425790748,
      "raw": {
       "metric": "accuracy",
       "value": 0.340632603406326,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.0904408049532455,
      "improvement_points": null,
      "runs": [
       "logic:knowline-2b_en",
       "math:knowline-2b_en",
       "logic:knowline-2b_en_dontknow",
       "math:knowline-2b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.052500000000000005,
      "gpu_cost_usd": 0.01117293586944372
     },
     {
      "repo": "fastino/GLiDE-0.8B",
      "commit": "9eacdeec79fa48ec52a8d6f54b331376b4bbfaf0",
      "model_key": "glide-0.8b",
      "name": "GLiDE 0.8B",
      "display_name": "GLiDE 0.8B",
      "tag": "",
      "class_label": "Open · 0.8B",
      "base_model": null,
      "setup_label": "GLiDE 0.8B (open)",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": 0.75,
      "params_active": 0.75,
      "quantization": "BF16",
      "score_0_100": 13.85340632603406,
      "raw": {
       "metric": "accuracy",
       "value": 0.3108272506082725,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.1713609660179691,
      "improvement_points": null,
      "runs": [
       "logic:glide-0.8b_en",
       "math:glide-0.8b_en",
       "logic:glide-0.8b_en_dontknow",
       "math:glide-0.8b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.021388888888888888,
      "gpu_cost_usd": 0.004551936835699293
     },
     {
      "repo": "startlux-models/StartLux-Decision-v1.1-0.8B-Instant",
      "commit": "6496ae641cc71b149ce9dad69bd97f2a44e7bf11",
      "model_key": "startlux-decision-v1.1-0.8b-instant",
      "name": "StartLux-Decision-v1.1-0.8B-Instant",
      "display_name": "StartLux-Decision-v1.1-0.8B-Instant",
      "tag": "",
      "class_label": "Open · 0.8B",
      "base_model": "Qwen3.5-0.8B",
      "setup_label": "StartLux-Decision-v1.1-0.8B-Instant",
      "maker": "StartLux",
      "kind": "decision",
      "params_total": 0.8,
      "params_active": 0.8,
      "quantization": "BF16",
      "score_0_100": 9.975669099756688,
      "raw": {
       "metric": "accuracy",
       "value": 0.2798053527980535,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.05794440174027441,
      "improvement_points": null,
      "runs": [
       "logic:startlux-0.8b_en",
       "math:startlux-0.8b_en",
       "logic:startlux-0.8b_en_dontknow",
       "math:startlux-0.8b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.025,
      "gpu_cost_usd": 0.005320445652116057
     },
     {
      "repo": "OmniJev/OneJev1.6-0.8B",
      "commit": "bcd88a382f314b26cb7b1b7f369c1ed3502ee7a0",
      "model_key": "onejev1.6-0.8b",
      "name": "OneJev1.6-0.8B",
      "display_name": "OneJev1.6-0.8B",
      "tag": "",
      "class_label": "Open · 0.8B",
      "base_model": null,
      "setup_label": "OneJev1.6-0.8B",
      "maker": "OmniJev",
      "kind": "decision",
      "params_total": 0.8,
      "params_active": 0.8,
      "quantization": "BF16",
      "score_0_100": 9.785583941605836,
      "raw": {
       "metric": "accuracy",
       "value": 0.2782846715328467,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.10419309610705596,
      "improvement_points": null,
      "runs": [
       "logic:onejev1.6-0.8b_en",
       "math:onejev1.6-0.8b_en",
       "logic:onejev1.6-0.8b_en_dontknow",
       "math:onejev1.6-0.8b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.017222222222222222,
      "gpu_cost_usd": 0.0036651958936799504
     },
     {
      "repo": "PelaAI/KnowLine-0.8B-Gen1",
      "commit": "6b27831f7b783c03278bd946d4d8f4f7a98a3798",
      "model_key": "knowline-0.8b-gen1",
      "name": "KnowLine-0.8B-Gen1",
      "display_name": "KnowLine-0.8B-Gen1",
      "tag": "",
      "class_label": "Open · 0.8B",
      "base_model": "Qwen3.5-0.8B",
      "setup_label": "KnowLine-0.8B-Gen1",
      "maker": "PelaAI",
      "kind": "decision",
      "params_total": 0.8,
      "params_active": 0.8,
      "quantization": "BF16",
      "score_0_100": 9.595498783454989,
      "raw": {
       "metric": "accuracy",
       "value": 0.2767639902676399,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.09799661221502888,
      "improvement_points": null,
      "runs": [
       "logic:knowline-0.8b_en",
       "math:knowline-0.8b_en",
       "logic:knowline-0.8b_en_dontknow",
       "math:knowline-0.8b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.027777777777777776,
      "gpu_cost_usd": 0.005911606280128953
     },
     {
      "repo": "StrandsAgents/strands-decider-2B-hobson-v21",
      "commit": "2b52a6235c1b8306bbfa30b00b9d4b74b63a39f5",
      "model_key": "strands-decider-2b",
      "name": "Strands Decider 2B",
      "display_name": "Strands Decider 2B",
      "tag": "",
      "class_label": "Open · 2.0B",
      "base_model": null,
      "setup_label": "Strands Decider 2B",
      "maker": "Strands Agents",
      "kind": "decision",
      "params_total": 2.0,
      "params_active": 2.0,
      "quantization": "BF16",
      "score_0_100": 8.150851581508515,
      "raw": {
       "metric": "accuracy",
       "value": 0.26520681265206814,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.09887791970802917,
      "improvement_points": null,
      "runs": [
       "logic:strands_en",
       "math:strands_en",
       "logic:strands_en_dontknow",
       "math:strands_en_dontknow"
      ],
      "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); the repository holds a LoRA and head on Qwen3.5-2B-Base",
      "gpu_hours": 0.03527777777777778,
      "gpu_cost_usd": 0.08784166666666667
     },
     {
      "repo": "knowledgator/gliclass-v4-edge",
      "commit": "885af1af8a07edf4ed34df83ca0d995ed8201283",
      "model_key": "gliclass-v4-edge",
      "name": "GLiClass v4 Edge",
      "display_name": "GLiClass v4 Edge",
      "tag": "",
      "class_label": "Open · 0.4B",
      "base_model": "DeBERTa-v3-large",
      "setup_label": "GLiClass v4 Edge",
      "maker": null,
      "kind": "decision",
      "params_total": 0.4,
      "params_active": 0.4,
      "quantization": "BF16",
      "score_0_100": 3.7028588807785887,
      "raw": {
       "metric": "accuracy",
       "value": 0.22962287104622872,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.07604377483198181,
      "improvement_points": null,
      "runs": [
       "logic:gliclass-v4-edge_en",
       "math:gliclass-v4-edge_en",
       "logic:gliclass-v4-edge_en_dontknow",
       "math:gliclass-v4-edge_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.015555555555555555,
      "gpu_cost_usd": 0.003310499516872213
     },
     {
      "repo": "fastino/GLiDE-0.4B",
      "commit": "6bda7ab91aeb36dcad6d27828a8871e548c7fc71",
      "model_key": "glide-0.4b",
      "name": "GLiDE 0.4B",
      "display_name": "GLiDE 0.4B",
      "tag": "",
      "class_label": "Open · 0.4B",
      "base_model": null,
      "setup_label": "GLiDE 0.4B (open)",
      "maker": "Fastino",
      "kind": "decision",
      "params_total": 0.4,
      "params_active": 0.4,
      "quantization": "BF16",
      "score_0_100": 1.8400243309002404,
      "raw": {
       "metric": "accuracy",
       "value": 0.21472019464720193,
       "questions": 3288,
       "options": 5
      },
      "calibration_error": 0.11006255383209179,
      "improvement_points": null,
      "runs": [
       "logic:glide-0.4b_en",
       "math:glide-0.4b_en",
       "logic:glide-0.4b_en_dontknow",
       "math:glide-0.4b_en_dontknow"
      ],
      "notes": "on our NVIDIA DGX Spark (GB10)",
      "gpu_hours": 0.012777777777777777,
      "gpu_cost_usd": 0.0027193388888593177
     }
    ]
   },
   "sha256": "628fd0ac978643e88c49ccb9cfd3288b85234915a723a4195ae17d9f0529d565"
  },
  "backgammon": {
   "url": "https://zekiworks.github.io/analysis/backgammon/bench-scores.json",
   "published": true,
   "seen": "2026-11-16",
   "file": {
    "bench": "backgammon",
    "models": [
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "chat",
      "maker": "OpenAI",
      "model_key": "gpt-6-astra",
      "name": "GPT-6 Astra",
      "notes": "League 3; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 3's field (different positions from the league's); scored against League 3's Random; ChatGPT subscription (Codex backend). Late entry to the original League 3 table (not in standings).; hosted (API or subscription); setting: high",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1489,
       "value": 8.82855723214838
      },
      "repo": null,
      "runs": [
       "backgammon:gpt-6-astra-high"
      ],
      "score_0_100": 92.5,
      "tag": "high"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "chat",
      "maker": "OpenAI",
      "model_key": "gpt-6.1-sol",
      "name": "GPT-6.1 Sol",
      "notes": "League 3; calibration definition pending Ümit; hosted (API or subscription); setting: high",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1389,
       "value": 9.89902464711812
      },
      "repo": null,
      "runs": [
       "backgammon:gpt-6-1-sol-high"
      ],
      "score_0_100": 91.59,
      "tag": "high"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": 8.47,
      "kind": "chat",
      "maker": "Google",
      "model_key": "gemini-4-argon",
      "name": "Gemini 4 Argon",
      "notes": "League 3; calibration definition pending Ümit; hosted (API or subscription); setting: high; route: Google Antigravity subscription; improvement_points: change in risks found, percentage points of risk-pays decisions (with lessons minus no memory); improvement_points from Run 2 after QC; original run +6.48, superseded",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1332,
       "value": 12.572338183720907
      },
      "repo": null,
      "runs": [
       "backgammon:gemini-4-argon-on"
      ],
      "score_0_100": 89.31,
      "tag": "high"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": 9.4,
      "kind": "chat",
      "maker": "Anthropic",
      "model_key": "claude-fable-5.1",
      "name": "Claude Fable 5.1",
      "notes": "League 3; calibration definition pending Ümit; hosted (API or subscription); setting: high; improvement_points: change in risks found, percentage points of risk-pays decisions (with lessons minus no memory)",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1286,
       "value": 13.954974595069144
      },
      "repo": null,
      "runs": [
       "backgammon:claude-fable-5-1-on"
      ],
      "score_0_100": 88.14,
      "tag": "high"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": 6.67,
      "kind": "chat",
      "maker": "Anthropic",
      "model_key": "claude-opus-5.5",
      "name": "Claude Opus 5.5",
      "notes": "League 3; calibration definition pending Ümit; hosted (API or subscription); setting: high; improvement_points: change in risks found, percentage points of risk-pays decisions (with lessons minus no memory)",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1477,
       "value": 14.443625438108425
      },
      "repo": null,
      "runs": [
       "backgammon:claude-opus-5-5-on"
      ],
      "score_0_100": 87.72,
      "tag": "high"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": 8.2,
      "kind": "chat",
      "maker": "Google",
      "model_key": "gemini-3.8-flash",
      "name": "Gemini 3.8 Flash",
      "notes": "League 2; calibration definition pending Ümit; hosted (API or subscription); setting: high; route: Google Antigravity subscription; improvement_points: change in risks found, percentage points of risk-pays decisions (with lessons minus no memory)",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2168,
       "value": 14.946429347585049
      },
      "repo": null,
      "runs": [
       "backgammon:gemini-3-8-flash-high"
      ],
      "score_0_100": 87.32,
      "tag": "high"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": 7.0,
      "kind": "chat",
      "maker": "Anthropic",
      "model_key": "claude-sonnet-5.5",
      "name": "Claude Sonnet 5.5",
      "notes": "League 2; calibration definition pending Ümit; hosted (API or subscription); setting: high; improvement_points: change in risks found, percentage points of risk-pays decisions (with lessons minus no memory)",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2164,
       "value": 16.719389183519066
      },
      "repo": null,
      "runs": [
       "backgammon:claude-sonnet-5-5-high"
      ],
      "score_0_100": 85.82,
      "tag": "high"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "chat",
      "maker": "xAI",
      "model_key": "grok-4.7",
      "name": "Grok 4.7",
      "notes": "League 3; calibration definition pending Ümit; hosted (API or subscription); setting: high",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1337,
       "value": 18.784194638591785
      },
      "repo": null,
      "runs": [
       "backgammon:grok-4-7-on"
      ],
      "score_0_100": 84.03,
      "tag": "high"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": 8.22,
      "kind": "chat",
      "maker": "DeepSeek",
      "model_key": "deepseek-v4.1-flash",
      "name": "DeepSeek-V4.1-Flash",
      "notes": "League 2; calibration definition pending Ümit; hosted (API or subscription); improvement_points: change in risks found, percentage points of risk-pays decisions (with lessons minus no memory); improvement_points from Run 2 after QC; original run +7.21, superseded",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2188,
       "value": 18.99183068894381
      },
      "repo": null,
      "runs": [
       "backgammon:deepseek-v4-1-flash-on"
      ],
      "score_0_100": 83.89,
      "tag": "low · API + NVIDIA RTX PRO 6000 Blackwell · first release"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": 8.47,
      "kind": "chat",
      "maker": "Google",
      "model_key": "gemma-4-31b",
      "name": "Gemma 4 31B",
      "notes": "League 2; calibration definition pending Ümit; run on our GPUs; setting: on; improvement_points: change in risks found, percentage points of risk-pays decisions (with lessons minus no memory)",
      "params_active": 31.27,
      "params_total": 31.27,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2122,
       "value": 19.575135612072078
      },
      "repo": null,
      "runs": [
       "backgammon:gemma-4-31b-on"
      ],
      "score_0_100": 83.4,
      "tag": "on · NVIDIA RTX PRO 6000 Blackwell + NVIDIA DGX Spark"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "chat",
      "maker": "Meta",
      "model_key": "muse-ember",
      "name": "Muse Ember",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 17 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Inception Mercury Decide: outside the requested scope (Ümit, 12 Nov); not played (opponent no longer served on OpenRouter); Meta's Muse Ember through OpenRouter, pinned to Meta (fallbacks off) for every decision; never on our GPUs. Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup); its matches were played on 10–11 Nov 2026. It played 17 of its 19 opponents: its pair with Mercury Decide was not played (Mercury Decide is no longer served on OpenRouter; match #932 stopped before any decision, unbilled; outside the requested scope, Ümit, 12 Nov) and its pair with GPT-6 Luna was not played (GPT-6 Luna's API was out of credit; outside the requested scope, Ümit, 12 Nov). Every request sent reasoning_effort \"max\"; 21.76 M of its 22.09 M output tokens were reasoning. The endpoint returned no token probabilities, so it answers with one letter, read from the reply; 46 of its 1,713 answers named no option and a random legal play was made. A quality check (12 Nov) found that 1 of those 46 was an OpenRouter provider error (no answer) scored as invalid, in match #926; the other 45 were real length cuts. A full Run 2 of match #926 (same seed, fixed runner 175141f, its own database) played 6 games to 0 against Random with 0 provider errors; the published numbers keep the original match. Each decision is billed at Meta's list price from the tokens OpenRouter reported.; hosted (API or subscription)",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1667,
       "value": 20.106377228692253
      },
      "repo": null,
      "runs": [
       "backgammon:muse-ember-max"
      ],
      "score_0_100": 83.4,
      "tag": "max"
     },
     {
      "calibration_error": null,
      "commit": "8e56aa1b236a6dd1064dcfd22ba1e5e360e4c140",
      "improvement_points": null,
      "kind": "decision",
      "maker": "PelaAI",
      "model_key": "knowline-31b-gen1.5",
      "name": "KnowLine-31B-Gen1.5",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 17 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Inception Mercury Decide: outside the requested scope (Ümit, 12 Nov); not played (opponent no longer served on OpenRouter); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup); its matches were played on 10–11 Nov 2026. It played 17 of its 19 opponents: its pair with Mercury Decide was not played (Mercury Decide is no longer served on OpenRouter; outside the requested scope, Ümit, 12 Nov) and its pair with GPT-6 Luna was not played (GPT-6 Luna's API was out of credit; outside the requested scope, Ümit, 12 Nov). Each option's probability is a softmax over the option labels, stored as returned. Its first run played 4 matches at once (#933–936) on a server meant for one request at a time; those matches were voided (rows kept, left out of every number) and the pairs replayed from the start, one match at a time.; run on our GPUs",
      "params_active": 31.27,
      "params_total": 31.27,
      "quantization": "BF16",
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1928,
       "value": 26.21519251019381
      },
      "repo": "PelaAI/KnowLine-31B-Gen1.5",
      "runs": [
       "backgammon:knowline-31b-gen1-5"
      ],
      "score_0_100": 78.35,
      "tag": "NVIDIA DGX Spark (GB10)"
     },
     {
      "calibration_error": null,
      "commit": "8fcee858f7f20a50bfa3c8f984c22adb3cbb46a2",
      "improvement_points": null,
      "kind": "decision",
      "maker": "Sthānika AI",
      "model_key": "sieve-31b-v3",
      "name": "Sieve 31B v3",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 17 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Inception Mercury Decide: outside the requested scope (Ümit, 12 Nov); not played (opponent no longer served on OpenRouter); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup), late-entry league #44. sthanika-ai/Sieve-31B-v3 @ 8fcee85 on our DGX Spark (spark2, the whole GB10, handed over by logic one server at a time), BF16, through the server its card names (sieve-serve, sieve-decisions 0.5.1 at github.com/sthanika-ai/Sieve 1e64657, defaults), one request at a time; never on a hosted API. First stage 12 Nov 2026: its 9 hosted and program opponents (#1221–#1229, runner a127626); local stage 14 Nov 2026: the 8 setups we run ourselves, one at a time on an RTX PRO 6000 Blackwell of ours (#1411–#1418, runner 2a22d1f). It played 17 of its 19 opponents: its pair with Mercury Decide was not played (Mercury Decide is no longer served on OpenRouter; match #1230 aborted at its warm-up; outside the requested scope, Ümit, 12 Nov) and its pair with GPT-6 Luna was not played (GPT-6 Luna's API was out of credit; outside the requested scope, Ümit, 12 Nov). Serving policy one read and 2 option orders averaged; probabilities rounded to 4 places. Its replies carry no input-token count, so the runner records 0 input tokens (per-token figures unknown, not zero). Never truncates (no token limit of its own; a request it cannot answer is refused, 422); nothing refused. Its first stage was stopped once after 12 h with pairs open (13 Nov 08:47 UTC) and finished at 11:29 UTC. Its cost is an estimate from its time on the DGX Spark (GPU time, at most).; run on our GPUs",
      "params_active": 31.27,
      "params_total": 31.27,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1740,
       "value": 27.491367377083876
      },
      "repo": "sthanika-ai/Sieve-31B-v3",
      "runs": [
       "backgammon:sieve-31b-v3"
      ],
      "score_0_100": 77.3,
      "tag": "NVIDIA DGX Spark (GB10)"
     },
     {
      "calibration_error": null,
      "commit": "fa4bf7ee3bb56a8bc421dad090631be1c5845c2d",
      "improvement_points": null,
      "kind": "decision",
      "maker": "REAX",
      "model_key": "reax-1.3",
      "name": "REAX 1.3",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 18 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup); its matches were played on 7 Nov 2026. Each option's probability is the 50/50 mean of the two parts' probabilities, stored as returned; in the smoke test before the run every probability sum was 1.000000. Its pair with GPT-6 Luna was not played: GPT-6 Luna's API was out of credit; the pair is outside the requested scope (Ümit, 12 Nov). Run note: the server's container ran without Docker's no-new-privileges setting, approved for this run only (the host's Docker refused every container with it); every other protection was kept: all capabilities dropped, read-only root filesystem, non-root user, weights mounted read-only, no outside network.; run on our GPUs",
      "params_active": 57.36,
      "params_total": 57.36,
      "quantization": "FP8",
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1998,
       "value": 27.772271060370826
      },
      "repo": "decisionmodels/REAX-1.3",
      "runs": [
       "backgammon:reax-1-3"
      ],
      "score_0_100": 77.07,
      "tag": "NVIDIA RTX PRO 6000 Blackwell"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "decision",
      "maker": "Google",
      "model_key": "gemini-decision-1",
      "name": "Gemini Decision 1",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 17 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Inception Mercury Decide: outside the requested scope (Ümit, 12 Nov); not played (opponent no longer served on OpenRouter); Google's Gemini Decision 1 through OpenRouter, pinned to Google AI Studio (fallbacks off) for every decision; never on our GPUs. Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup); its matches were played on 11 Nov 2026. It played 17 of its 19 opponents: its pair with Mercury Decide was not played (Mercury Decide is no longer served on OpenRouter; match #1016 aborted; outside the requested scope, Ümit, 12 Nov) and its pair with GPT-6 Luna was not played (GPT-6 Luna's API was out of credit; outside the requested scope, Ümit, 12 Nov). Two runner builds: dab6da5 for its first 9 matches (#993–#1013, before 21:46:31 UTC), with no probability-map check, and 38024d6 (map-check v2) from 21:46:31 UTC for its last 8 (#1053–#1063). Under map-check v2 a returned probability map is used only if it covers exactly the options shown, every value is finite and in [0, 1], and the values sum to 1 within 1e-6; this sum rule is new (no frozen sum tolerance existed), was chosen on 11 Nov 2026 and applies to this entry only; a failed map still plays the returned choice and stores no probabilities. An offline audit of the first matches under the same rule found no failing map. Option order: we send the options in our seeded order, as for every setup, while Google's docs say the API sorts the option keys alphabetically (UTF-8 byte order; an exact tie goes to the alphabetically first key), so the order the model sees is probably not ours (Google's docs; not verified by us). Its cost is the billed cost of each decision; the entry, with its paid opponents, billed $1.1819 in all.; hosted (API or subscription)",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1704,
       "value": 27.968576685153824
      },
      "repo": null,
      "runs": [
       "backgammon:gemini-decision-1"
      ],
      "score_0_100": 76.91,
      "tag": "20261111"
     },
     {
      "calibration_error": null,
      "commit": "61cec44c281c693992e3d6ebeb44dd3402dfa186",
      "improvement_points": null,
      "kind": "decision",
      "maker": "Base Labs",
      "model_key": "mammoth-1",
      "name": "Mammoth-1",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup); its matches were played on 2 Nov 2026. Probabilities come rounded to 6 decimals by the loader and are stored as returned.; run on our GPUs",
      "params_active": 27.36,
      "params_total": 27.36,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2021,
       "value": 28.5432201794974
      },
      "repo": "baselabs/mammoth-1",
      "runs": [
       "backgammon:base-labs-mammoth-1"
      ],
      "score_0_100": 76.43,
      "tag": "NVIDIA RTX PRO 6000 Blackwell"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": 4.8,
      "kind": "chat",
      "maker": "Moonshot",
      "model_key": "kimi-k3.1-pro",
      "name": "Kimi K3.1 Pro",
      "notes": "League 3; calibration definition pending Ümit; hosted (API or subscription); setting: off; improvement_points: change in risks found, percentage points of risk-pays decisions (with lessons minus no memory)",
      "params_active": 104,
      "params_total": 2800,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1167,
       "value": 28.04553088665826
      },
      "repo": null,
      "runs": [
       "backgammon:kimi-k3-1-pro-off"
      ],
      "score_0_100": 76.16,
      "tag": "off"
     },
     {
      "calibration_error": null,
      "commit": "866c0fc6e040cba23fcc72113a80e5c99d5ebab1",
      "improvement_points": null,
      "kind": "decision",
      "maker": "Turion.AI",
      "model_key": "turion-jev-31b",
      "name": "Turion Jev 31B",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 17 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Inception Mercury Decide: outside the requested scope (Ümit, 12 Nov); not played (opponent no longer served on OpenRouter); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup), late-entry league #46. turion-ai/turion-jev-31b @ 866c0fc on our DGX Spark (spark2, the whole GB10, handed over by logic one server at a time), FP8 at load, through the server its package ships (python -m turion_jev.serve, vLLM 0.32.0 in process, --quantization fp8 --no-recheck), one request at a time; never on a hosted API. First stage 13 Nov 2026: its 9 hosted and program opponents (#1241–#1249, runner a127626 for #1241, then a72223d); local stage 14 Nov 2026: the 8 setups we run ourselves, one at a time on an RTX PRO 6000 Blackwell of ours (#1403–#1410, runner 2a22d1f). It played 17 of its 19 opponents: its pair with Mercury Decide was not played (Mercury Decide is no longer served on OpenRouter; match #1250 aborted at its warm-up; outside the requested scope, Ümit, 12 Nov) and its pair with GPT-6 Luna was not played (GPT-6 Luna's API was out of credit; outside the requested scope, Ümit, 12 Nov). BF16 weights quantised to FP8 at load and re-scoring off (--no-recheck), as in the card's run; probabilities rounded to 6 places. Its server was stopped after an OOM kill on spark2 (13 Nov 12:33 UTC, 3 failed health checks); from 14:29 UTC it ran through logic's launcher turion_serve_headroom.py --gpu-mem 0.80 (logic #6191), which sets only vLLM's memory share (the KV cache size) and then runs turion_jev.serve unchanged, pinned by its sha256. Never truncates: a prompt over 65,536 tokens is refused (422); nothing refused. Its cost is an estimate from its time on the DGX Spark (GPU time, at most).; run on our GPUs",
      "params_active": 31.27,
      "params_total": 31.27,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1855,
       "value": 29.695650621887165
      },
      "repo": "turion-ai/turion-jev-31b",
      "runs": [
       "backgammon:turion-jev-31b"
      ],
      "score_0_100": 75.48,
      "tag": "NVIDIA DGX Spark (GB10)"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": 19.0,
      "kind": "chat",
      "maker": "Alibaba",
      "model_key": "qwen3.8-27b",
      "name": "Qwen3.8-27B",
      "notes": "League 2; calibration definition pending Ümit; run on our GPUs; setting: low; improvement_points: change in risks found, percentage points of risk-pays decisions (with lessons minus no memory)",
      "params_active": 27.78,
      "params_total": 27.78,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2108,
       "value": 29.032554555210716
      },
      "repo": null,
      "runs": [
       "backgammon:qwen3-8-27b-on"
      ],
      "score_0_100": 75.38,
      "tag": "low · NVIDIA RTX PRO 6000 Blackwell + NVIDIA DGX Spark"
     },
     {
      "calibration_error": null,
      "commit": "cac65e1982a0237e996419ba559004522d9a33ac",
      "improvement_points": null,
      "kind": "decision",
      "maker": "Nace.AI",
      "model_key": "drex-max-v1",
      "name": "Drex Max",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 18 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup); its matches were played on 6 Nov 2026. Probabilities are stored as returned. Its pair with GPT-6 Luna was not played: GPT-6 Luna's API was out of credit; the pair is outside the requested scope (Ümit, 12 Nov).; run on our GPUs",
      "params_active": 26.09,
      "params_total": 26.09,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2148,
       "value": 30.399406949679058
      },
      "repo": "nace-ai/drex-max-v1",
      "runs": [
       "backgammon:drex-max-v1"
      ],
      "score_0_100": 74.9,
      "tag": "NVIDIA RTX PRO 6000 Blackwell"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "decision",
      "maker": "Nace.AI",
      "model_key": "drex-max-1.1",
      "name": "Drex Max v1.1",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 18 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup); its matches were played on 6 Nov 2026. Probabilities are stored as returned. Its pair with GPT-6 Luna was not played: GPT-6 Luna's API was out of credit; the pair is outside the requested scope (Ümit, 12 Nov).; hosted (API or subscription)",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2154,
       "value": 30.938308412760858
      },
      "repo": null,
      "runs": [
       "backgammon:drex-max-1-1"
      ],
      "score_0_100": 74.45,
      "tag": ""
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "decision",
      "maker": "Liquid AI",
      "model_key": "liquid-d1-ultra",
      "name": "d1-ultra",
      "notes": "League 2; calibration definition pending Ümit; hosted (API or subscription)",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2071,
       "value": 30.33737738255658
      },
      "repo": null,
      "runs": [
       "backgammon:liquid-d1-ultra"
      ],
      "score_0_100": 74.27,
      "tag": ""
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": 17.62,
      "kind": "chat",
      "maker": "Z.ai",
      "model_key": "glm-5.3",
      "name": "GLM-5.3",
      "notes": "League 2; calibration definition pending Ümit; hosted (API or subscription); setting: low; improvement_points: change in risks found, percentage points of risk-pays decisions (with lessons minus no memory)",
      "params_active": null,
      "params_total": 753.33,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1989,
       "value": 30.866173822236096
      },
      "repo": null,
      "runs": [
       "backgammon:glm-5-3-on"
      ],
      "score_0_100": 73.82,
      "tag": "low"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "decision",
      "maker": "TypeSafe",
      "model_key": "jev-1.14.0",
      "name": "Jev 1.14",
      "notes": "League 2; calibration definition pending Ümit; hosted (API or subscription)",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1912,
       "value": 31.73452527671429
      },
      "repo": null,
      "runs": [
       "backgammon:jev-1-14-0"
      ],
      "score_0_100": 73.09,
      "tag": ""
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "decision",
      "maker": "Fastino",
      "model_key": "fastino-glide",
      "name": "GLiDE",
      "notes": "League 2; calibration definition pending Ümit; hosted (API or subscription)",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2159,
       "value": 33.1826399685664
      },
      "repo": null,
      "runs": [
       "backgammon:fastino-glide"
      ],
      "score_0_100": 71.86,
      "tag": "API"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "decision",
      "maker": "TypeSafe",
      "model_key": "jev-1.13.0",
      "name": "Jev 1.13",
      "notes": "League 1; calibration definition pending Ümit; hosted (API or subscription)",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1911,
       "value": 34.409363666858304
      },
      "repo": null,
      "runs": [
       "backgammon:jev-1-13-0"
      ],
      "score_0_100": 71.59,
      "tag": ""
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": 2.67,
      "kind": "chat",
      "maker": "Alibaba",
      "model_key": "qwen3.8-flash",
      "name": "Qwen3.8 Flash",
      "notes": "League 2; calibration definition pending Ümit; hosted (API or subscription); improvement_points: change in risks found, percentage points of risk-pays decisions (with lessons minus no memory)",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2027,
       "value": 33.577662334397445
      },
      "repo": null,
      "runs": [
       "backgammon:qwen3-8-flash-off"
      ],
      "score_0_100": 71.52,
      "tag": "off"
     },
     {
      "calibration_error": null,
      "commit": "a7e45896",
      "improvement_points": null,
      "kind": "decision",
      "maker": "Fastino",
      "model_key": "fastino-glide-v1.2",
      "name": "GLiDE",
      "notes": "League 1; calibration definition pending Ümit; run on our GPUs",
      "params_active": 26.09,
      "params_total": 26.09,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2003,
       "value": 35.004573431065005
      },
      "repo": "fastino/GLiDE",
      "runs": [
       "backgammon:fastino-glide-v1-2-local"
      ],
      "score_0_100": 71.1,
      "tag": "v1.2 · NVIDIA RTX PRO 6000 Blackwell"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "decision",
      "maker": "Celeris",
      "model_key": "celeris-1-decision",
      "name": "celeris-1-decision",
      "notes": "League 1; calibration definition pending Ümit; hosted (API or subscription)",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1922,
       "value": 35.64279046564767
      },
      "repo": null,
      "runs": [
       "backgammon:celeris-1-decision"
      ],
      "score_0_100": 70.57,
      "tag": ""
     },
     {
      "calibration_error": null,
      "commit": "225c913ddc8761a8264f04f64548e1f97e443eb3",
      "improvement_points": null,
      "kind": "decision",
      "maker": "Perplexity",
      "model_key": "perplexity-decider-27b-v1.2",
      "name": "Perplexity Decider 27B v1.2",
      "notes": "League 1; calibration definition pending Ümit; run on our GPUs",
      "params_active": 26.09,
      "params_total": 26.09,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2028,
       "value": 36.933414013486875
      },
      "repo": "perplexity-ai/pplx-decider-v1.2",
      "runs": [
       "backgammon:decider-v1-2-27b"
      ],
      "score_0_100": 69.5,
      "tag": "NVIDIA RTX PRO 6000 Blackwell"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "decision",
      "maker": "Cloudflare",
      "model_key": "clef",
      "name": "Clef",
      "notes": "League 2; calibration definition pending Ümit; run on our GPUs",
      "params_active": 27.48,
      "params_total": 27.48,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1978,
       "value": 36.536419001399445
      },
      "repo": null,
      "runs": [
       "backgammon:clef"
      ],
      "score_0_100": 69.02,
      "tag": "NVIDIA RTX PRO 6000 Blackwell + NVIDIA DGX Spark"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "decision",
      "maker": "Nace.AI",
      "model_key": "drex-2.0",
      "name": "Drex 2.0",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 18 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup); its matches were played on 6 Nov 2026. Probabilities come rounded to 4 decimals by Nace's API and are stored as returned. Its pair with GPT-6 Luna was not played: GPT-6 Luna's API was out of credit; the pair is outside the requested scope (Ümit, 12 Nov).; hosted (API or subscription)",
      "params_active": 3,
      "params_total": 35.11,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1937,
       "value": 38.43995843373074
      },
      "repo": null,
      "runs": [
       "backgammon:drex-2-0"
      ],
      "score_0_100": 68.26,
      "tag": "API"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "decision",
      "maker": "Blockbrain Labs",
      "model_key": "cygnet",
      "name": "Cygnet",
      "notes": "League 1; calibration definition pending Ümit; run on our GPUs",
      "params_active": 11.96,
      "params_total": 11.96,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2016,
       "value": 38.550341649660986
      },
      "repo": null,
      "runs": [
       "backgammon:cygnet-gemma-4-12b"
      ],
      "score_0_100": 68.17,
      "tag": "NVIDIA RTX PRO 6000 Blackwell + NVIDIA DGX Spark"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "decision",
      "maker": "Perplexity",
      "model_key": "perplexity-decider-27b-v1.1",
      "name": "Perplexity Decider 27B v1.1",
      "notes": "League 1; calibration definition pending Ümit; run on our GPUs; route: our GPUs (released checkpoint); the other benches' row is the Perplexity Decisions API",
      "params_active": 26.09,
      "params_total": 26.09,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2052,
       "value": 38.782911052016026
      },
      "repo": null,
      "runs": [
       "backgammon:decider-v1-1-27b"
      ],
      "score_0_100": 67.98,
      "tag": "NVIDIA RTX PRO 6000 Blackwell + NVIDIA DGX Spark"
     },
     {
      "calibration_error": null,
      "commit": "416088f77cfce09633906a3675c7cd2e5dabcf10",
      "improvement_points": null,
      "kind": "decision",
      "maker": "StartLux",
      "model_key": "startlux-decision-v1.1-4b-instant",
      "name": "StartLux-Decision-v1.1-4B-Instant",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 17 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Inception Mercury Decide: outside the requested scope (Ümit, 12 Nov); not played (opponent no longer served on OpenRouter); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup), late-entry league #47. startlux-models/StartLux-Decision-v1.1-4B-Instant @ 416088f on our DGX Spark (spark2, the whole GB10, handed over by logic one server at a time), BF16, through its package's own server (startlux-decide 1.1.0 serve --launch-vllm, vLLM 0.31.0), one request at a time; never on a hosted API. First stage 13 Nov 2026: its 9 hosted and program opponents (#1251–#1259, runner 62ae46a); local stage 14 Nov 2026: the 8 setups we run ourselves, one at a time on an RTX PRO 6000 Blackwell of ours (#1387–#1394, runner 2a22d1f). It played 17 of its 19 opponents: its pair with Mercury Decide was not played (Mercury Decide is no longer served on OpenRouter; match #1260 aborted at its warm-up; outside the requested scope, Ümit, 12 Nov) and its pair with GPT-6 Luna was not played (GPT-6 Luna's API was out of credit; outside the requested scope, Ümit, 12 Nov). Instant: one read-out per pass, no thinking; 26 letters per pass including the hidden 'None of the above' (serve's default none_check); the server knocks longer lists out in groups of at most 25. Never truncates: vLLM refuses over 131,072 tokens (422); nothing refused. Its cost is an estimate from its time on the DGX Spark (GPU time, at most).; run on our GPUs",
      "params_active": 4.66,
      "params_total": 4.66,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1989,
       "value": 40.14492664971263
      },
      "repo": "startlux-models/StartLux-Decision-v1.1-4B-Instant",
      "runs": [
       "backgammon:startlux-decision-v1-1-4b-instant"
      ],
      "score_0_100": 66.85,
      "tag": "NVIDIA DGX Spark (GB10)"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "decision",
      "maker": "Subquadratic",
      "model_key": "subq-snap-1.1",
      "name": "SubQ Snap 1.1",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup); its matches were played on 2 Nov 2026. Probabilities arrive as 32-bit floats and are stored as returned (they sum to 1 only within 32-bit rounding).; hosted (API or subscription)",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2326,
       "value": 40.19971192523722
      },
      "repo": null,
      "runs": [
       "backgammon:subq-snap-1-1"
      ],
      "score_0_100": 66.81,
      "tag": ""
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "decision",
      "maker": "OpenAI",
      "model_key": "gpt-6-luna",
      "name": "GPT-6 Luna",
      "notes": "League 1; calibration definition pending Ümit; hosted (API or subscription)",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1962,
       "value": 43.31066665998171
      },
      "repo": null,
      "runs": [
       "backgammon:gpt-6-luna"
      ],
      "score_0_100": 64.24,
      "tag": "Decisions API"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "decision",
      "maker": "Subquadratic",
      "model_key": "subq-snap-preview",
      "name": "SubQ Snap Preview",
      "notes": "League 1; calibration definition pending Ümit; hosted (API or subscription)",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2200,
       "value": 43.870584172281355
      },
      "repo": null,
      "runs": [
       "backgammon:subq-snap-preview"
      ],
      "score_0_100": 63.77,
      "tag": ""
     },
     {
      "calibration_error": null,
      "commit": "b3f8a93462fb1051a9de031e94a7ff380e254072",
      "improvement_points": null,
      "kind": "decision",
      "maker": "Base Labs",
      "model_key": "elephant-1",
      "name": "Elephant-1",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup); its matches were played on 2 Nov 2026. Probabilities come rounded to 6 decimals by the loader and are stored as returned.; run on our GPUs",
      "params_active": 9.41,
      "params_total": 9.41,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2044,
       "value": 44.56163451454644
      },
      "repo": "baselabs/elephant-1",
      "runs": [
       "backgammon:base-labs-elephant-1"
      ],
      "score_0_100": 63.2,
      "tag": "NVIDIA RTX PRO 6000 Blackwell"
     },
     {
      "calibration_error": null,
      "commit": "10e7671d44e4b37700020b1eed3f594c5f2af0b4",
      "improvement_points": null,
      "kind": "decision",
      "maker": "Liquid AI",
      "model_key": "liquid-d1-2b",
      "name": "d1-2B",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 17 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Inception Mercury Decide: outside the requested scope (Ümit, 12 Nov); not played (opponent no longer served on OpenRouter); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup), late-entry league #52. LiquidAI/d1-2B @ 10e7671 on one RTX PRO 6000 Blackwell of ours (GPU 2), BF16, through logic's d1_server.py: the command and environment of logic's decision/servers.sh d1-2b (torch 2.15.0+cu132, transformers 5.22.1), the card's call model.system_one(state, questions) unchanged, model field LiquidAI/d1-2B; one request at a time; never on a hosted API. First stage 14 Nov 2026, 16:12–16:16 UTC: its 9 hosted and program opponents (#1419–#1427, runner 8ebad40); local stage 14 Nov 2026, 18:44–19:03 UTC: the 8 setups we run ourselves, one at a time on the same card (#1429–#1436, runner 46c984e, which adds only d1-Omni-2B's entry beside 8ebad40's d1-2B client). It played 17 of its 19 opponents: its pair with Mercury Decide was not played (Mercury Decide is no longer served on OpenRouter; match #1428 aborted at its warm-up; outside the requested scope, Ümit, 12 Nov) and its pair with GPT-6 Luna was not played (GPT-6 Luna's API was out of credit; outside the requested scope, Ümit, 12 Nov). Every option count gets distinct single-token codes (A..IU at 255). Context 128,000 positions; nothing in its code shortens a prompt, and a request it cannot answer is refused (400), never cut: League 1's longest stored request (251 options) is 20,759 tokens with its tokenizer and prompt; nothing refused. Its cost is an estimate from its time on the RTX PRO 6000 (GPU time, at most).; run on our GPUs",
      "params_active": 1.92,
      "params_total": 1.92,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1980,
       "value": 47.10251993753693
      },
      "repo": "LiquidAI/d1-2B",
      "runs": [
       "backgammon:liquid-d1-2b"
      ],
      "score_0_100": 61.11,
      "tag": "NVIDIA RTX PRO 6000 Blackwell"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "chat",
      "maker": "Liquid AI",
      "model_key": "lfm2.5-24b-a2b",
      "name": "LFM2.5-24B-A2B",
      "notes": "League 1; calibration definition pending Ümit; run on our GPUs",
      "params_active": 2.45,
      "params_total": 23.97,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2193,
       "value": 47.686285635297615
      },
      "repo": null,
      "runs": [
       "backgammon:lfm2-5-24b-a2b-on"
      ],
      "score_0_100": 60.62,
      "tag": "on · NVIDIA RTX PRO 6000 Blackwell + NVIDIA DGX Spark"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "decision",
      "maker": "Inception",
      "model_key": "inception-mercury-decide",
      "name": "Mercury Decide",
      "notes": "League 1; calibration definition pending Ümit; hosted (API or subscription)",
      "params_active": null,
      "params_total": null,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1853,
       "value": 47.94801757969987
      },
      "repo": null,
      "runs": [
       "backgammon:mercury-decide"
      ],
      "score_0_100": 60.41,
      "tag": ""
     },
     {
      "calibration_error": null,
      "commit": "f02a632fbe3a4698cd1ff5c0b929ea240e2b8eda",
      "improvement_points": null,
      "kind": "decision",
      "maker": "Apertix AI",
      "model_key": "cerno-9b-v1.2",
      "name": "Cerno 9B v1.2",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 17 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Inception Mercury Decide: outside the requested scope (Ümit, 12 Nov); not played (opponent no longer served on OpenRouter); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup), late-entry league #48. apertixai/cerno-9b-v1.2 @ f02a632 on our DGX Spark (spark2, the whole GB10, handed over by logic one server at a time), BF16, through the card's own in-process transformers engine (CernoEngine) behind logic's thin System One server (cerno_server.py, request/response mapping only), one request at a time; never on a hosted API. First stage 13 Nov 2026: its 9 hosted and program opponents (#1261–#1269, runner 2a22d1f); local stage 14 Nov 2026: the 8 setups we run ourselves, one at a time on an RTX PRO 6000 Blackwell of ours (#1395–#1402, runner 2a22d1f). It played 17 of its 19 opponents: its pair with Mercury Decide was not played (Mercury Decide is no longer served on OpenRouter; match #1270 aborted at its warm-up; outside the requested scope, Ümit, 12 Nov) and its pair with GPT-6 Luna was not played (GPT-6 Luna's API was out of credit; outside the requested scope, Ümit, 12 Nov). Probabilities rounded to 4 places. The decider would cut a state ('Context:' block) over 32,768 tokens (its engine default) without refusing; that cut is never reached in League 1 (longest state block 5,856 tokens over its 31,793 distinct stored requests); the longest prompt (251 options) is 28,349 tokens, within its 262,144 context. Its cost is an estimate from its time on the DGX Spark (GPU time, at most).; run on our GPUs",
      "params_active": 8.95,
      "params_total": 8.95,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2108,
       "value": 48.46733851637514
      },
      "repo": "apertixai/cerno-9b-v1.2",
      "runs": [
       "backgammon:cerno-9b-v1-2"
      ],
      "score_0_100": 59.98,
      "tag": "NVIDIA DGX Spark (GB10)"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "decision",
      "maker": "EldanRing",
      "model_key": "winnow-12b",
      "name": "Winnow-12B",
      "notes": "League 1; calibration definition pending Ümit; run on our GPUs",
      "params_active": 11.91,
      "params_total": 11.91,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2015,
       "value": 49.727740965173496
      },
      "repo": null,
      "runs": [
       "backgammon:winnow-12b"
      ],
      "score_0_100": 58.94,
      "tag": "NVIDIA RTX PRO 6000 Blackwell"
     },
     {
      "calibration_error": null,
      "commit": "81222dae8bf8bdd7f903643b5b2a8803bf2e3176",
      "improvement_points": null,
      "kind": "decision",
      "maker": "Liquid AI",
      "model_key": "liquid-d1-omni-2b",
      "name": "d1-Omni-2B",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 17 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Inception Mercury Decide: outside the requested scope (Ümit, 12 Nov); not played (opponent no longer served on OpenRouter); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup), late-entry league #53. LiquidAI/d1-Omni-2B @ 81222da, d1-2B with audio added (the card: text, pictures and audio in), on one RTX PRO 6000 Blackwell of ours (GPU 2, the card d1-2B used, after it), BF16, through logic's d1_server.py: the command of logic's decision/servers.sh d1-omni-2b in d1-2B's environment (torch 2.15.0+cu132, transformers 5.22.1), the card's call model.system_one(state, questions) unchanged, model field LiquidAI/d1-Omni-2B; text only: backgammon positions are text, so its audio and picture inputs are unused; one request at a time; never on a hosted API. Its request and limits are d1-2B's: the model's request code is byte-identical to d1-2B's and its tokenizer only adds two audio tokens, so text prompts tokenize the same. First stage 14 Nov 2026, 19:47–19:50 UTC: its 9 hosted and program opponents (#1437–#1445); local stage 14 Nov 2026, 19:56–20:15 UTC: the 8 setups we run ourselves, one at a time on the same card (#1447–#1454); runner 46c984e throughout. It played 17 of its 19 opponents: its pair with Mercury Decide was not played (Mercury Decide is no longer served on OpenRouter; match #1446 aborted at its warm-up; outside the requested scope, Ümit, 12 Nov) and its pair with GPT-6 Luna was not played (GPT-6 Luna's API was out of credit; outside the requested scope, Ümit, 12 Nov). Every option count gets distinct single-token codes (A..IU at 255). Context 128,000 positions; nothing in its code shortens a prompt, and a request it cannot answer is refused (400), never cut: League 1's longest stored request (251 options) is 20,759 tokens; nothing refused. Its cost is an estimate from its time on the RTX PRO 6000 (GPU time, at most).; run on our GPUs",
      "params_active": 2.17,
      "params_total": 2.17,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1710,
       "value": 49.79613642943533
      },
      "repo": "LiquidAI/d1-Omni-2B",
      "runs": [
       "backgammon:liquid-d1-omni-2b"
      ],
      "score_0_100": 58.88,
      "tag": "NVIDIA RTX PRO 6000 Blackwell"
     },
     {
      "calibration_error": null,
      "commit": "f1e6cd147dda82c9ffa3e8e725d5c1f175566fb1",
      "improvement_points": null,
      "kind": "decision",
      "maker": "StartLux",
      "model_key": "startlux-decision-v1.1-2b-instant",
      "name": "StartLux-Decision-v1.1-2B-Instant",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 17 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Inception Mercury Decide: outside the requested scope (Ümit, 12 Nov); not played (opponent no longer served on OpenRouter); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup), late-entry league #23. startlux-models/StartLux-Decision-v1.1-2B-Instant @ f1e6cd1 on our DGX Spark (spark2, the whole GB10, handed over by logic one server at a time), BF16, through its package's own server (startlux-decide 1.1.0 serve --launch-vllm, vLLM 0.31.0), one request at a time; never on a hosted API. First stage 11 Nov 2026: its 9 hosted and program opponents (#970–#978, runner 2a27d5d); local stage 14 Nov 2026: the 8 setups we run ourselves, one at a time on an RTX PRO 6000 Blackwell of ours (#1379–#1386, runner 2a27d5d). It played 17 of its 19 opponents: its pair with Mercury Decide was not played (Mercury Decide is no longer served on OpenRouter; match #979 aborted at its warm-up; outside the requested scope, Ümit, 12 Nov) and its pair with GPT-6 Luna was not played (GPT-6 Luna's API was out of credit; outside the requested scope, Ümit, 12 Nov). Instant: one read-out per pass, no thinking; one letter per option, 26 per pass; the server knocks longer lists out in groups of at most 25 (the 3 best of each to a final pass); serve's default none_check (a hidden 'None of the above' option). Its cost is an estimate from its time on the DGX Spark (GPU time, at most).; run on our GPUs",
      "params_active": 2.27,
      "params_total": 2.27,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1771,
       "value": 50.041675172169825
      },
      "repo": "startlux-models/StartLux-Decision-v1.1-2B-Instant",
      "runs": [
       "backgammon:startlux-decision-v1-1-2b"
      ],
      "score_0_100": 58.68,
      "tag": "NVIDIA DGX Spark (GB10)"
     },
     {
      "calibration_error": null,
      "commit": "779944d2cc8ffa19cef386aee83c83bcb967ade2",
      "improvement_points": null,
      "kind": "decision",
      "maker": "PelaAI",
      "model_key": "knowline-4b-gen5.5",
      "name": "KnowLine 4B Gen5.5",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 17 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Inception Mercury Decide: outside the requested scope (Ümit, 12 Nov); not played (opponent no longer served on OpenRouter); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup), late-entry league #22. PelaAI/KnowLine-4B-Gen5.5 @ 779944d on our DGX Spark (spark2, the whole GB10, handed over by logic one server at a time), FP8 at load, through the repo's own serve_knowline.sh (SGLang 0.5.21 with FP8 at load, as the card serves it, then its knowline_server.py front end), one request at a time; never on a hosted API. First stage 11 Nov 2026: its 9 hosted and program opponents (#957–#965, runner b727222); local stage 14 Nov 2026: the 8 setups we run ourselves, one at a time on an RTX PRO 6000 Blackwell of ours (#1371–#1378, runner b727222). It played 17 of its 19 opponents: its pair with Mercury Decide was not played (Mercury Decide is no longer served on OpenRouter; match #966 aborted at its warm-up; outside the requested scope, Ümit, 12 Nov) and its pair with GPT-6 Luna was not played (GPT-6 Luna's API was out of credit; outside the requested scope, Ümit, 12 Nov). FP8 at load (--quantization fp8, as the card and serve_knowline.sh serve it), --mem-fraction-static 0.72. Its cost is an estimate from its time on the DGX Spark (GPU time, at most).; run on our GPUs",
      "params_active": 4.66,
      "params_total": 4.66,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2153,
       "value": 53.58964786854003
      },
      "repo": "PelaAI/KnowLine-4B-Gen5.5",
      "runs": [
       "backgammon:knowline-4b-gen5-5"
      ],
      "score_0_100": 55.75,
      "tag": "NVIDIA DGX Spark (GB10)"
     },
     {
      "calibration_error": null,
      "commit": "53245d470a4ce316e32fd0d18c24e142c697f40b",
      "improvement_points": null,
      "kind": "decision",
      "maker": "PelaAI",
      "model_key": "knowline-2b-gen1",
      "name": "KnowLine-2B-Gen1",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 17 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Inception Mercury Decide: outside the requested scope (Ümit, 12 Nov); not played (opponent no longer served on OpenRouter); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup), late-entry league #40. PelaAI/KnowLine-2B-Gen1 @ 53245d4 on our DGX Spark (spark2, the whole GB10, handed over by logic one server at a time), BF16, through the repo's own serve_knowline.sh (SGLang 0.5.21, then its knowline_server.py front end), one request at a time; never on a hosted API. First stage 12 Nov 2026: its 9 hosted and program opponents (#1181–#1189, runner e4ce0e9); local stage 14 Nov 2026: the 8 setups we run ourselves, one at a time on an RTX PRO 6000 Blackwell of ours (#1355–#1362, runner 2a22d1f). It played 17 of its 19 opponents: its pair with Mercury Decide was not played (Mercury Decide is no longer served on OpenRouter; match #1190 aborted at its warm-up; outside the requested scope, Ümit, 12 Nov) and its pair with GPT-6 Luna was not played (GPT-6 Luna's API was out of credit; outside the requested scope, Ümit, 12 Nov). It never truncates (SGLang without --allow-auto-truncate refuses over its 262,144-token context); nothing refused. Its cost is an estimate from its time on the DGX Spark (GPU time, at most).; run on our GPUs",
      "params_active": 2.27,
      "params_total": 2.27,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1955,
       "value": 54.54787061647381
      },
      "repo": "PelaAI/KnowLine-2B-Gen1",
      "runs": [
       "backgammon:knowline-2b-gen1"
      ],
      "score_0_100": 54.96,
      "tag": "NVIDIA DGX Spark (GB10)"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "decision",
      "maker": "Perplexity",
      "model_key": "perplexity-decider-flash-v1.2",
      "name": "Perplexity Decider Flash v1.2",
      "notes": "League 1; calibration definition pending Ümit; run on our GPUs",
      "params_active": 4.54,
      "params_total": 4.54,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2006,
       "value": 59.642184061459744
      },
      "repo": null,
      "runs": [
       "backgammon:decider-flash-v1-2"
      ],
      "score_0_100": 50.75,
      "tag": "NVIDIA RTX PRO 6000 Blackwell"
     },
     {
      "calibration_error": null,
      "commit": "6496ae641cc71b149ce9dad69bd97f2a44e7bf11",
      "improvement_points": null,
      "kind": "decision",
      "maker": "StartLux",
      "model_key": "startlux-decision-v1.1-0.8b-instant",
      "name": "StartLux-Decision-v1.1-0.8B-Instant",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 17 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Inception Mercury Decide: outside the requested scope (Ümit, 12 Nov); not played (opponent no longer served on OpenRouter); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup), late-entry league #27. startlux-models/StartLux-Decision-v1.1-0.8B-Instant @ 6496ae6 on our DGX Spark (spark2, the whole GB10, handed over by logic one server at a time), BF16, through its package's own server (startlux-decide 1.1.0 serve --launch-vllm, vLLM 0.31.0), one request at a time; never on a hosted API. First stage 11 Nov 2026: its 9 hosted and program opponents (#1000–#1014, runner e4ce0e9); local stage 14 Nov 2026: the 8 setups we run ourselves, one at a time on an RTX PRO 6000 Blackwell of ours (#1339–#1346, runner 2a22d1f). It played 17 of its 19 opponents: its pair with Mercury Decide was not played (Mercury Decide is no longer served on OpenRouter; match #1015 aborted at its warm-up; outside the requested scope, Ümit, 12 Nov) and its pair with GPT-6 Luna was not played (GPT-6 Luna's API was out of credit; outside the requested scope, Ümit, 12 Nov). Instant: one read-out per pass, no thinking; 26 letters per pass including the hidden 'None of the above' (serve's default none_check); the server knocks longer lists out in groups of at most 25. Never truncates: vLLM refuses over 131,072 tokens (422); nothing refused. Its cost is an estimate from its time on the DGX Spark (GPU time, at most).; run on our GPUs",
      "params_active": 0.87,
      "params_total": 0.87,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1882,
       "value": 64.7133435424532
      },
      "repo": "startlux-models/StartLux-Decision-v1.1-0.8B-Instant",
      "runs": [
       "backgammon:startlux-decision-v1-1-0-8b-instant"
      ],
      "score_0_100": 46.56,
      "tag": "NVIDIA DGX Spark (GB10)"
     },
     {
      "calibration_error": null,
      "commit": null,
      "improvement_points": null,
      "kind": "decision",
      "maker": "Zefan Cai",
      "model_key": "open-jev-27b-v1.1",
      "name": "Open-Jev-27B-v1.1",
      "notes": "League 1; calibration definition pending Ümit; run on our GPUs",
      "params_active": 27.8,
      "params_total": 27.8,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 2002,
       "value": 65.55707858397172
      },
      "repo": null,
      "runs": [
       "backgammon:open-jev-27b-v1-1"
      ],
      "score_0_100": 45.87,
      "tag": "NVIDIA RTX PRO 6000 Blackwell"
     },
     {
      "calibration_error": null,
      "commit": "00f122c120cdeed82284106cb77d9bab5a50ad16",
      "improvement_points": null,
      "kind": "decision",
      "maker": "Fastino",
      "model_key": "glide-2b",
      "name": "GLiDE 2B",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 17 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Inception Mercury Decide: outside the requested scope (Ümit, 12 Nov); not played (opponent no longer served on OpenRouter); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup), late-entry league #41. fastino/GLiDE-2B @ 00f122c on our DGX Spark (spark2, the whole GB10, handed over by logic one server at a time), BF16, through the server its repository ships (python -m glide.server, GLIDE_MAX_BATCH=1), one request at a time; never on a hosted API. First stage 12 Nov 2026: its 9 hosted and program opponents (#1191–#1199, runner e4ce0e9); local stage 13 Nov, 14 Nov 2026: the 8 setups we run ourselves, one at a time on an RTX PRO 6000 Blackwell of ours (#1298–#1305, runner 2a22d1f). It played 17 of its 19 opponents: its pair with Mercury Decide was not played (Mercury Decide is no longer served on OpenRouter; match #1200 aborted at its warm-up; outside the requested scope, Ümit, 12 Nov) and its pair with GPT-6 Luna was not played (GPT-6 Luna's API was out of credit; outside the requested scope, Ümit, 12 Nov). It never truncates: a prompt over GLIDE_MAX_LEN 40,000 tokens is refused (422); League 1's longest stored request (251 options) is 28,323 tokens; nothing refused. Its cost is an estimate from its time on the DGX Spark (GPU time, at most).; run on our GPUs",
      "params_active": 1.88,
      "params_total": 1.88,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1676,
       "value": 67.35153148630923
      },
      "repo": "fastino/GLiDE-2B",
      "runs": [
       "backgammon:glide-2b"
      ],
      "score_0_100": 44.39,
      "tag": "NVIDIA DGX Spark (GB10)"
     },
     {
      "calibration_error": null,
      "commit": "6b27831f7b783c03278bd946d4d8f4f7a98a3798",
      "improvement_points": null,
      "kind": "decision",
      "maker": "PelaAI",
      "model_key": "knowline-0.8b-gen1",
      "name": "KnowLine-0.8B-Gen1",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 17 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Inception Mercury Decide: outside the requested scope (Ümit, 12 Nov); not played (opponent no longer served on OpenRouter); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup), late-entry league #25. PelaAI/KnowLine-0.8B-Gen1 @ 6b27831 on our DGX Spark (spark2, the whole GB10, handed over by logic one server at a time), BF16, through the repo's own serve_knowline.sh (SGLang 0.5.21, then its knowline_server.py front end), one request at a time; never on a hosted API. First stage 11 Nov 2026: its 9 hosted and program opponents (#983–#991, runner e4ce0e9); local stage 14 Nov 2026: the 8 setups we run ourselves, one at a time on an RTX PRO 6000 Blackwell of ours (#1331–#1338, runner 2a22d1f). It played 17 of its 19 opponents: its pair with Mercury Decide was not played (Mercury Decide is no longer served on OpenRouter; match #992 aborted at its warm-up; outside the requested scope, Ümit, 12 Nov) and its pair with GPT-6 Luna was not played (GPT-6 Luna's API was out of credit; outside the requested scope, Ümit, 12 Nov). It never truncates (SGLang without --allow-auto-truncate refuses over its 262,144-token context); nothing refused. Its cost is an estimate from its time on the DGX Spark (GPU time, at most).; run on our GPUs",
      "params_active": 0.87,
      "params_total": 0.87,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1728,
       "value": 71.94117152270067
      },
      "repo": "PelaAI/KnowLine-0.8B-Gen1",
      "runs": [
       "backgammon:knowline-0-8b-gen1"
      ],
      "score_0_100": 40.6,
      "tag": "NVIDIA DGX Spark (GB10)"
     },
     {
      "calibration_error": null,
      "commit": "9eacdeec79fa48ec52a8d6f54b331376b4bbfaf0",
      "improvement_points": null,
      "kind": "decision",
      "maker": "Fastino",
      "model_key": "glide-0.8b",
      "name": "GLiDE 0.8B",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 17 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Inception Mercury Decide: outside the requested scope (Ümit, 12 Nov); not played (opponent no longer served on OpenRouter); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup), late-entry league #42. fastino/GLiDE-0.8B @ 9eacdee on our DGX Spark (spark2, the whole GB10, handed over by logic one server at a time), BF16, through the server its repository ships (python -m glide.server, GLIDE_MAX_BATCH=1), one request at a time; never on a hosted API. First stage 12 Nov 2026: its 9 hosted and program opponents (#1201–#1209, runner e4ce0e9); local stage 14 Nov 2026: the 8 setups we run ourselves, one at a time on an RTX PRO 6000 Blackwell of ours (#1315–#1322, runner 2a22d1f). It played 17 of its 19 opponents: its pair with Mercury Decide was not played (Mercury Decide is no longer served on OpenRouter; match #1210 aborted at its warm-up; outside the requested scope, Ümit, 12 Nov) and its pair with GPT-6 Luna was not played (GPT-6 Luna's API was out of credit; outside the requested scope, Ümit, 12 Nov). It never truncates: a prompt over GLIDE_MAX_LEN 40,000 tokens is refused (422); League 1's longest stored request (251 options) is 28,323 tokens; nothing refused. Its cost is an estimate from its time on the DGX Spark (GPU time, at most).; run on our GPUs",
      "params_active": 0.75,
      "params_total": 0.75,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1832,
       "value": 80.36733062441692
      },
      "repo": "fastino/GLiDE-0.8B",
      "runs": [
       "backgammon:glide-0-8b"
      ],
      "score_0_100": 33.64,
      "tag": "NVIDIA DGX Spark (GB10)"
     },
     {
      "calibration_error": null,
      "commit": "bcd88a382f314b26cb7b1b7f369c1ed3502ee7a0",
      "improvement_points": null,
      "kind": "decision",
      "maker": "OmniJev",
      "model_key": "onejev1.6-0.8b",
      "name": "OneJev1.6-0.8B",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 17 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Inception Mercury Decide: outside the requested scope (Ümit, 12 Nov); not played (opponent no longer served on OpenRouter); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup), late-entry league #30. OmniJev/OneJev1.6-0.8B @ bcd88a3 on our DGX Spark (spark2, the whole GB10, handed over by logic one server at a time), BF16, through the server its card names (Flide 0.1.11 flide serve, format onejev-slot-v2), one request at a time; never on a hosted API. First stage 11 Nov 2026: its 9 hosted and program opponents (#1041–#1049, runner e4ce0e9); local stage 14 Nov 2026: the 8 setups we run ourselves, one at a time on an RTX PRO 6000 Blackwell of ours (#1347–#1354, runner 2a22d1f). It played 17 of its 19 opponents: its pair with Mercury Decide was not played (Mercury Decide is no longer served on OpenRouter; match #1050 aborted at its warm-up; outside the requested scope, Ümit, 12 Nov) and its pair with GPT-6 Luna was not played (GPT-6 Luna's API was out of credit; outside the requested scope, Ümit, 12 Nov). No calibration file shipped, so temperature 1; probabilities rounded to 6 places by the server. Never truncates: state plus a question over 32,768 tokens, or a request over 65,536, is refused (422); nothing refused. Its cost is an estimate from its time on the DGX Spark (GPU time, at most).; run on our GPUs",
      "params_active": 0.85,
      "params_total": 0.85,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1763,
       "value": 82.51933623327157
      },
      "repo": "OmniJev/OneJev1.6-0.8B",
      "runs": [
       "backgammon:onejev1-6-0-8b"
      ],
      "score_0_100": 31.86,
      "tag": "NVIDIA DGX Spark (GB10)"
     },
     {
      "calibration_error": null,
      "commit": "6bda7ab91aeb36dcad6d27828a8871e548c7fc71",
      "improvement_points": null,
      "kind": "decision",
      "maker": "Fastino",
      "model_key": "glide-0.4b",
      "name": "GLiDE 0.4B",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 17 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Inception Mercury Decide: outside the requested scope (Ümit, 12 Nov); not played (opponent no longer served on OpenRouter); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup), late-entry league #43. fastino/GLiDE-0.4B @ 6bda7ab on our DGX Spark (spark2, the whole GB10, handed over by logic one server at a time), BF16, through the server its repository ships (python -m glide.server, GLIDE_MAX_BATCH=1, GLIDE_MAX_LEN=8006), one request at a time; never on a hosted API. First stage 12 Nov 2026: its 9 hosted and program opponents (#1211–#1219, runner e4ce0e9); local stage 14 Nov 2026: the 8 setups we run ourselves, one at a time on an RTX PRO 6000 Blackwell of ours (#1323–#1330, runner 2a22d1f). It played 17 of its 19 opponents: its pair with Mercury Decide was not played (Mercury Decide is no longer served on OpenRouter; match #1220 aborted at its warm-up; outside the requested scope, Ümit, 12 Nov) and its pair with GPT-6 Luna was not played (GPT-6 Luna's API was out of credit; outside the requested scope, Ümit, 12 Nov). An encoder (ModernBERT) whose model would shorten a prompt over 7,999 tokens instead of refusing it, so it gets at most 74 options per request: every League 1 stored request of at most 74 options fits (at most 7,833 tokens with the card's prompt), and longer play lists are split into equal groups of at most 74 and a final among the group winners (the league's split rule, as Winnow and Clef; 678 of League 1's 36,995 decisions, 1.8%, have more than 74 options). The server runs with GLIDE_MAX_LEN=8006, which refuses (422) any prompt the model would shorten, as a backstop. Nothing shortened or refused. Its cost is an estimate from its time on the DGX Spark (GPU time, at most).; run on our GPUs",
      "params_active": 0.4,
      "params_total": 0.4,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1355,
       "value": 113.21918488208658
      },
      "repo": "fastino/GLiDE-0.4B",
      "runs": [
       "backgammon:glide-0-4b"
      ],
      "score_0_100": 6.51,
      "tag": "NVIDIA DGX Spark (GB10)"
     },
     {
      "calibration_error": null,
      "commit": "885af1af8a07edf4ed34df83ca0d995ed8201283",
      "improvement_points": null,
      "kind": "decision",
      "maker": "Knowledgator",
      "model_key": "gliclass-v4-edge",
      "name": "GLiClass v4 Edge",
      "notes": "League 1; calibration definition pending Ümit; late entry: its own decisions in late-entry matches against League 1's field (different positions from the league's); scored against League 1's Random; played 17 of 19 matches; GPT-6 Luna (Decisions API): outside the requested scope (Ümit, 12 Nov); not played (opponent's API out of credit); Inception Mercury Decide: outside the requested scope (Ümit, 12 Nov); not played (opponent no longer served on OpenRouter); Late entry to the original League 1 table (not in standings, not eligible for the Hype Cup), late-entry league #36. knowledgator/gliclass-v4-edge @ 885af1a on our DGX Spark (spark2, the whole GB10, handed over by logic one server at a time), BF16, through turkce's gliclass_server.py with the card's own System One mapping (gliclass 0.1.22, one pass per choice question), one request at a time; never on a hosted API. First stage 12 Nov 2026: its 9 hosted and program opponents (#1141–#1169, runner e4ce0e9); local stage 14 Nov 2026: the 8 setups we run ourselves, one at a time on an RTX PRO 6000 Blackwell of ours (#1363–#1370, runner 2a22d1f). It played 17 of its 19 opponents: its pair with Mercury Decide was not played (Mercury Decide is no longer served on OpenRouter; match #1170 aborted at its warm-up; outside the requested scope, Ümit, 12 Nov) and its pair with GPT-6 Luna was not played (GPT-6 Luna's API was out of credit; outside the requested scope, Ümit, 12 Nov). Options, instructions and state share one 16,384-token input (refused with 422 above it, never cut); League 1's worst case, the longest state plus 218 options at the longest label, is 16,367 tokens, so play lists over 218 options are split into equal groups of at most 218 and a final among the group winners (the league's split rule, as Winnow and Clef; 24 of League 1's 36,995 decisions have more than 218 options). Nothing shortened or refused. Its first stage was interrupted once (12 Nov 08:59 UTC, the server failed 3 health checks in a row) and resumed at 10:15 UTC. Its cost is an estimate from its time on the DGX Spark (GPU time, at most).; run on our GPUs",
      "params_active": 0.44,
      "params_total": 0.44,
      "quantization": null,
      "raw": {
       "metric": "error rate (PR, lower is better)",
       "options": null,
       "questions": 1405,
       "value": 133.5217322955352
      },
      "repo": "knowledgator/gliclass-v4-edge",
      "runs": [
       "backgammon:gliclass-v4-edge"
      ],
      "score_0_100": -10.25,
      "tag": "NVIDIA DGX Spark (GB10)"
     }
    ],
    "score": {
     "hundred": "100 = error rate 0: the engine's best play on every decision",
     "zero": "0 = a random legal play's error rate in the same league (League 1: 121.1, League 2: 117.9, League 3: 117.6; late entries: the league whose field they played); below 0 = worse than a random legal play. score = 100 × (1 − error rate / Random's error rate in that league); error rate (PR) = 500 × mean GNUbg equity loss per decision"
    },
    "title": "Backgammon",
    "updated": "2026-11-16T21:14Z",
    "url": "https://zekiworks.github.io/analysis/backgammon/"
   },
   "sha256": "b3a42ca9b2ccefb4d058acc06cd0dc3e16f6ab5ba8ab89ec9953a49148bd32d2"
  },
  "grading": {
   "url": "https://zekiworks.github.io/analysis/grading/bench-scores.json",
   "published": true,
   "seen": "2026-11-16",
   "file": {
    "bench": "grading",
    "title": "Answer Check · does the text give this answer?",
    "url": "https://zekiworks.github.io/analysis/grading/",
    "updated": "2026-11-16T19:44:15Z",
    "score": {
     "zero": "uniform chance: one of the three labels at random on every pair (accuracy 1/3)",
     "hundred": "every one of the 2,069 pairs labelled right (accuracy 100%)"
    },
    "models": [
     {
      "name": "karpo-decision-31B",
      "display_name": "karpo-decision-31B",
      "tag": "",
      "model_key": "karpo-decision-31b",
      "maker": "MachinePulse",
      "kind": "decision",
      "repo": "MachinePulseAI/karpo-decision-31B",
      "commit": "d69cdd57eab23271cbe65236ca8a00d80d7bb8c7",
      "params_total": 31.3,
      "params_active": 31.3,
      "quantization": "BF16",
      "score_0_100": 89.99516674722086,
      "raw": {
       "metric": "accuracy",
       "value": 0.9333011116481392,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.02798799607249211,
      "improvement_points": null,
      "runs": [
       "grading:site-karpo-decision-31b-v1"
      ],
      "notes": "MachinePulseAI/karpo-decision-31B@d69cdd57, its card's karpo-decision package (machinepulse-ai/karpo-decision@ac40d399) System One server, BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 3 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
      "gpu_hours": 0.13244746838548202,
      "gpu_cost_usd": null
     },
     {
      "name": "Copernicus-31B",
      "display_name": "Copernicus-31B",
      "tag": "",
      "model_key": "copernicus-31b",
      "maker": "Poindexter Labs",
      "kind": "decision",
      "repo": "Poindexter-Labs/Copernicus-31B",
      "commit": "be715259e97a743341f2d7a3e6a6298dab23ca0e",
      "params_total": 31.3,
      "params_active": 31.3,
      "quantization": "BF16",
      "score_0_100": 88.9801836636056,
      "raw": {
       "metric": "accuracy",
       "value": 0.9265345577573707,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.10967730780108835,
      "improvement_points": null,
      "runs": [
       "grading:site-copernicus-31b-v1"
      ],
      "notes": "Poindexter-Labs/Copernicus-31B@be715259, its card's copernicus_server.py (Transformers backend, the card's defaults), BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 3 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
      "gpu_hours": 0.33160791070381207,
      "gpu_cost_usd": null
     },
     {
      "name": "KnowLine-31B-Gen1.5",
      "display_name": "KnowLine-31B-Gen1.5",
      "tag": "",
      "model_key": "knowline-31b-gen1.5",
      "maker": "PelaAI",
      "kind": "decision",
      "repo": "PelaAI/KnowLine-31B-Gen1.5",
      "commit": "8e56aa1b236a6dd1064dcfd22ba1e5e360e4c140",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "BF16",
      "score_0_100": 88.83518608023199,
      "raw": {
       "metric": "accuracy",
       "value": 0.9255679072015467,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.05716040274263797,
      "improvement_points": null,
      "runs": [
       "grading:site-knowline-31b-gen1.5-v1"
      ],
      "notes": "PelaAI/KnowLine-31B-Gen1.5@8e56aa1b, the card's serve_knowline.sh (SGLang), BF16, on one DGX Spark (ours) as for its published runs, self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 1 published-cohort pair was sent in test requests whose outputs were not kept, so its scored row comes from a later request",
      "gpu_hours": 1.5019916954069576,
      "gpu_cost_usd": null
     },
     {
      "name": "Sieve 31B v3",
      "display_name": "Sieve 31B v3",
      "tag": "",
      "model_key": "sieve-31b-v3",
      "maker": "Sthānika AI",
      "kind": "decision",
      "repo": "sthanika-ai/Sieve-31B-v3",
      "commit": "8fcee858f7f20a50bfa3c8f984c22adb3cbb46a2",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "BF16",
      "score_0_100": 88.11019816336393,
      "raw": {
       "metric": "accuracy",
       "value": 0.9207346544224263,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.0700594490091832,
      "improvement_points": null,
      "runs": [
       "grading:site-sieve-31b-v3-v1"
      ],
      "notes": "sthanika-ai/Sieve-31B-v3@8fcee858, sieve-serve 0.5.1 (text only, design 'letters'), BF16, on one DGX Spark (ours) as for its published runs, self-hosted; the server reports no input-token count; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 2 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
      "gpu_hours": 1.6228648508469032,
      "gpu_cost_usd": null
     },
     {
      "name": "KnowLine-31B-Gen2",
      "display_name": "KnowLine-31B-Gen2",
      "tag": "",
      "model_key": "knowline-31b-gen2",
      "maker": "PelaAI",
      "kind": "decision",
      "repo": "PelaAI/KnowLine-31B-Gen2",
      "commit": "fd99a9ed2255d8a67dd47a44cc429e638f611593",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "BF16",
      "score_0_100": 87.6752054132431,
      "raw": {
       "metric": "accuracy",
       "value": 0.917834702754954,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.07166560874597089,
      "improvement_points": null,
      "runs": [
       "grading:site-knowline-31b-gen2-v1"
      ],
      "notes": "PelaAI/KnowLine-31B-Gen2@fd99a9ed, its card's serve_knowline.sh (SGLang, then knowline_server.py, card defaults) in the lmsysorg/sglang image with SGLang 0.5.23 (the card names 0.5.21, which we do not have), BF16, SHA256SUMS checked, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); the run overlapped a host GPU recovery warning; its serving process continued, and the run completed without interruption or resume (producer report)",
      "gpu_hours": 0.2582788412532403,
      "gpu_cost_usd": null
     },
     {
      "name": "REAX 1.3",
      "display_name": "REAX 1.3",
      "tag": "",
      "model_key": "reax-1.3",
      "maker": "REAX",
      "kind": "decision",
      "repo": "decisionmodels/REAX-1.3",
      "commit": "fa4bf7ee3bb56a8bc421dad090631be1c5845c2d",
      "params_total": 57.36,
      "params_active": 57.36,
      "quantization": "FP8",
      "score_0_100": 86.95021749637505,
      "raw": {
       "metric": "accuracy",
       "value": 0.9130014499758338,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.048274518773097234,
      "improvement_points": null,
      "runs": [
       "grading:site-reax-1.3-v1"
      ],
      "notes": "decisionmodels/REAX-1.3@fa4bf7ee, REAX Engine reax13 server on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 3 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
      "gpu_hours": 0.42833667592136754,
      "gpu_cost_usd": null
     },
     {
      "name": "d1.2-ultra",
      "display_name": "d1.2-ultra",
      "tag": "",
      "model_key": "liquid-d1.2-ultra",
      "maker": "Liquid AI",
      "kind": "decision",
      "repo": null,
      "commit": null,
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 86.44272595456741,
      "raw": {
       "metric": "accuracy",
       "value": 0.9096181730304495,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.08786327021452686,
      "improvement_points": null,
      "runs": [
       "grading:site-liquid-d1.2-ultra-v1"
      ],
      "notes": "Liquid AI decisions API (api.liquid.ai/decisions, model d1.2-ultra); one question per request, 8 requests at a time (latency not comparable across different concurrency)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "name": "Turion Jev 31B",
      "display_name": "Turion Jev 31B",
      "tag": "",
      "model_key": "turion-jev-31b",
      "maker": "Turion.AI",
      "kind": "decision",
      "repo": "turion-ai/turion-jev-31b",
      "commit": "866c0fc6e040cba23fcc72113a80e5c99d5ebab1",
      "params_total": 31.0,
      "params_active": 31.0,
      "quantization": "FP8",
      "score_0_100": 86.44272595456741,
      "raw": {
       "metric": "accuracy",
       "value": 0.9096181730304495,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.06694250072498793,
      "improvement_points": null,
      "runs": [
       "grading:site-turion-jev-31b-v1"
      ],
      "notes": "turion-ai/turion-jev-31b@866c0fc6, the card's turion_jev.serve on vLLM 0.32.0 (BF16 weights, FP8 at load, text only), on one DGX Spark (ours) as for its published runs, self-hosted; vLLM memory share 0.80 instead of the card's 0.92 (KV cache only); one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 2 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
      "gpu_hours": 0.7637274554815272,
      "gpu_cost_usd": null
     },
     {
      "name": "Perplexity Decider 27B v1.3",
      "display_name": "Perplexity Decider 27B v1.3",
      "tag": "",
      "model_key": "perplexity-decider-27b-v1.3",
      "maker": "Perplexity",
      "kind": "decision",
      "repo": "perplexity-ai/pplx-decider-v1.3",
      "commit": "9c2ebb27470d3dec2ad7ab727727e40d142c7213",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 86.22522957950699,
      "raw": {
       "metric": "accuracy",
       "value": 0.9081681971967134,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.02761915331616796,
      "improvement_points": null,
      "runs": [
       "grading:site-perplexity-decider-27b-v1.3-v1"
      ],
      "notes": "perplexity-ai/pplx-decider-v1.3@9c2ebb27, the runner's pplx_decider_server.py with the checkpoint's own pplx_decider package (as for its published v1.2 runs), BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 3 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
      "gpu_hours": 0.14082144462229776,
      "gpu_cost_usd": null
     },
     {
      "name": "Inkling 2 Small",
      "display_name": "Inkling 2 Small",
      "tag": "thinking on (no off mode) · chat",
      "model_key": "inkling-2-small",
      "maker": "Thinking Machines",
      "kind": "chat",
      "repo": null,
      "commit": null,
      "params_total": 594.0,
      "params_active": 17.0,
      "quantization": null,
      "score_0_100": 86.22522957950699,
      "raw": {
       "metric": "accuracy",
       "value": 0.9081681971967134,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "grading:chat-inkling-2-small-v1"
      ],
      "notes": "OpenRouter thinkingmachines/inkling-2-small (paid per token; providers with no data retention and no training on inputs), reasoning at the route's default; billed per token at OpenRouter's list price; one question per request, 1 request at a time (latency not comparable across different concurrency); list prices $0.30 per 1M input and $1.00 per 1M output tokens (OpenRouter, 16 Nov); answered by OpenRouter's providers Together 1,104, Cloudflare 317, PrimeIntellect 242, Nebius 201, DeepInfra 179, Modal 26; OpenRouter's reported charges for the 2,069 billed requests of this model: $2.8921 (actual billing, distinct from the list-price figure); thinking on; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 0 of 2069 replies (0%) had no answer marker and count wrong; every reply kept",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "name": "GLiDE 27B",
      "display_name": "GLiDE 27B",
      "tag": "",
      "model_key": "glide-27b",
      "maker": "Fastino",
      "kind": "decision",
      "repo": "fastino/GLiDE-27B",
      "commit": "4d32743ae96b74f0116f050d37742bbd43c04e7d",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 86.0802319961334,
      "raw": {
       "metric": "accuracy",
       "value": 0.9072015466408894,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.03608106551078754,
      "improvement_points": null,
      "runs": [
       "grading:site-glide-27b-v1"
      ],
      "notes": "fastino/GLiDE-27B@4d32743a, GLiDE server (glide.server), BF16, GLIDE_MAX_BATCH=1, on one DGX Spark (ours) as for its published runs, self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 2 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
      "gpu_hours": 0.6924089500718801,
      "gpu_cost_usd": null
     },
     {
      "name": "Logits Sigma-1 Duo",
      "display_name": "Logits Sigma-1 Duo",
      "tag": "",
      "model_key": "logits-sigma-1-duo",
      "maker": "Logits",
      "kind": "decision",
      "repo": "logitshq/Logits-Sigma-1-Duo",
      "commit": "a0c1b6e8242916aafdcf9a5a49c1a952cb56e091",
      "params_total": 57.36,
      "params_active": 57.36,
      "quantization": "FP8",
      "score_0_100": 85.13774770420494,
      "raw": {
       "metric": "accuracy",
       "value": 0.9009183180280329,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.07037908459683984,
      "improvement_points": null,
      "runs": [
       "grading:site-logits-sigma-1-duo-v1"
      ],
      "notes": "logitshq/Logits-Sigma-1-Duo@a0c1b6e8 (weights and code identical to d9641cd8, the revision its card's run names; two members averaged per option, FP8 weights), its card's serving image (code/Dockerfile: torch 2.15, transformers 5.22.1, causal-conv1d 1.7.0, kit decision-index@25c6ec7) plus only gcc, libc6-dev and python3-dev, which its Triton kernels need to compile a launcher at first use and the image lacks, with the engine and option its serve.sh fixes (FP8 matmul on its own Triton kernel: its cuBLAS block-scaled path was not taken on this GPU, a fallback the engine makes itself), behind di_serve.py, our System One wrapper of the engine, on one RTX PRO 6000 (ours), self-hosted; text only (the load report lists FP8 scales missing for member A's vision tower, which text requests do not use; not checked on images); one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.41566869665931905,
      "gpu_cost_usd": null
     },
     {
      "name": "GLiDE",
      "display_name": "GLiDE",
      "tag": "API",
      "model_key": "fastino-glide",
      "maker": "Fastino",
      "kind": "decision",
      "repo": null,
      "commit": null,
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 85.06524891251811,
      "raw": {
       "metric": "accuracy",
       "value": 0.9004349927501208,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.02970154943215885,
      "improvement_points": null,
      "runs": [
       "grading:site-fastino-glide-v1"
      ],
      "notes": "Fastino decisions API (api.fastino.ai, model fastino/GLiDE); one question per request, 8 requests at a time (latency not comparable across different concurrency)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "name": "SubQ Snap 1.1",
      "display_name": "SubQ Snap 1.1",
      "tag": "",
      "model_key": "subq-snap-1.1",
      "maker": "Subquadratic",
      "kind": "decision",
      "repo": null,
      "commit": null,
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 84.92025132914449,
      "raw": {
       "metric": "accuracy",
       "value": 0.8994683421942967,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.02284575354194461,
      "improvement_points": null,
      "runs": [
       "grading:site-subq-snap-1.1-v1"
      ],
      "notes": "SubQ decisions API (api.subq.ai, model subq-snap-1.1); its probabilities were byte-identical to SubQ Snap Preview's on all 2,069 pairs; one question per request, 8 requests at a time (latency not comparable across different concurrency)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "name": "GLiDE",
      "display_name": "GLiDE",
      "tag": "v1.2",
      "model_key": "fastino-glide-v1.2",
      "maker": "Fastino",
      "kind": "decision",
      "repo": "fastino/GLiDE",
      "commit": "a7e45896be7c96ba3354891da790c5ec99c30b6c",
      "params_total": 27.0,
      "params_active": 27.0,
      "quantization": "BF16",
      "score_0_100": 84.92025132914449,
      "raw": {
       "metric": "accuracy",
       "value": 0.8994683421942967,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.04729810505945925,
      "improvement_points": null,
      "runs": [
       "grading:site-fastino-glide-v1.2-v1"
      ],
      "notes": "fastino/GLiDE@a7e4589, GLiDE server (Fastino's release code), BF16, GLIDE_MAX_BATCH=1, on one RTX PRO 6000 (ours) as the backgammon league's pinned server, self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 2 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
      "gpu_hours": 0.14576463736400147,
      "gpu_cost_usd": null
     },
     {
      "name": "d1-ultra",
      "display_name": "d1-ultra",
      "tag": "",
      "model_key": "liquid-d1-ultra",
      "maker": "Liquid AI",
      "kind": "decision",
      "repo": null,
      "commit": null,
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 84.70275495408409,
      "raw": {
       "metric": "accuracy",
       "value": 0.8980183663605606,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.11420895215834384,
      "improvement_points": null,
      "runs": [
       "grading:d1-ultra-v1"
      ],
      "notes": "Liquid AI decisions API (api.liquid.ai); one question per request, 8 requests at a time (latency not comparable across different concurrency)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "name": "Microsoft-Decision-1",
      "display_name": "Microsoft-Decision-1",
      "tag": "",
      "model_key": "microsoft-decision-1",
      "maker": "Microsoft",
      "kind": "decision",
      "repo": null,
      "commit": null,
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 83.90526824552923,
      "raw": {
       "metric": "accuracy",
       "value": 0.8927017883035283,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.023815195630622524,
      "improvement_points": null,
      "runs": [
       "grading:site-microsoft-decision-1-v1"
      ],
      "notes": "OpenRouter decisions API (openrouter.ai/api, model microsoft/microsoft-decision-1); one question per request, 8 requests at a time (latency not comparable across different concurrency)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "name": "vera-medium-omni",
      "display_name": "vera-medium-omni",
      "tag": "",
      "model_key": "vera-medium-omni",
      "maker": "scar-ai",
      "kind": "decision",
      "repo": "scar-ai/vera-medium-omni",
      "commit": "1f23a6cf5304e8e53b7ded53980765d5f8f4047c",
      "params_total": 4.2,
      "params_active": 4.2,
      "quantization": "BF16",
      "score_0_100": 83.68777187046882,
      "raw": {
       "metric": "accuracy",
       "value": 0.8912518124697921,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.12360604156597389,
      "improvement_points": null,
      "runs": [
       "grading:site-vera-medium-omni-v1"
      ],
      "notes": "scar-ai/vera-medium-omni@1f23a6cf, its card's own server (vera-serve from vera-s1 0.1.8, PyTorch engine), BF16, on one RTX PRO 6000 (ours), self-hosted; server probabilities are rounded to four decimal places and retained without renormalization; ECE uses the reported maximum probability; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.03423223886573668,
      "gpu_cost_usd": null
     },
     {
      "name": "Gemini 4 Hydrogen",
      "display_name": "Gemini 4 Hydrogen",
      "tag": "thinking on (no off mode) · chat",
      "model_key": "gemini-4-hydrogen",
      "maker": "Google",
      "kind": "chat",
      "repo": null,
      "commit": null,
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 83.68777187046882,
      "raw": {
       "metric": "accuracy",
       "value": 0.8912518124697921,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "grading:chat-gemini-4-hydrogen-v1"
      ],
      "notes": "Google Antigravity agent harness through Can's agbench bridge (Google AI Ultra subscription), thinking high, tool descriptions present and tools denied, a fresh conversation per request; $0 billed (the subscription's quota); one question per request, 1 request at a time (latency not comparable across different concurrency); the pass stopped once when the bridge's safety switch tripped on a false positive (fixed by its owner) and resumed in the same order by a recorded decision; the one request in flight at the stop was sent again, so its recorded outcome is not guaranteed to be the model's first output; thinking on; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 0 of 2069 replies (0%) had no answer marker and count wrong; every reply kept",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "name": "Jev 1.14",
      "display_name": "Jev 1.14",
      "tag": "",
      "model_key": "jev-1.14.0",
      "maker": "TypeSafe",
      "kind": "decision",
      "repo": null,
      "commit": null,
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 83.61527307878202,
      "raw": {
       "metric": "accuracy",
       "value": 0.8907684871918802,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.027994200096665072,
      "improvement_points": null,
      "runs": [
       "grading:site-jev-1.14.0-v1"
      ],
      "notes": "TypeSafe decisions API (api.typesafe.ai, model jev-1.14.0); one question per request, 8 requests at a time (latency not comparable across different concurrency)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "name": "Jev 1.13",
      "display_name": "Jev 1.13",
      "tag": "",
      "model_key": "jev-1.13.0",
      "maker": "TypeSafe",
      "kind": "decision",
      "repo": null,
      "commit": null,
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 82.96278395360076,
      "raw": {
       "metric": "accuracy",
       "value": 0.8864185596906718,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.018777187046882544,
      "improvement_points": null,
      "runs": [
       "grading:site-jev-1.13.0-v1"
      ],
      "notes": "TypeSafe decisions API (api.typesafe.ai, model jev-1.13.0); one question per request, 8 requests at a time (latency not comparable across different concurrency)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "name": "Jeffan-9B-1211",
      "display_name": "Jeffan-9B-1211",
      "tag": "",
      "model_key": "jeffan-9b-1211",
      "maker": "tattrongvu",
      "kind": "decision",
      "repo": "tattrongvu/Jeffan-9B-1211",
      "commit": "862d1a7a8b64a8d708ddb0b0cd775a740f01841a",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 82.96278395360076,
      "raw": {
       "metric": "accuracy",
       "value": 0.8864185596906718,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.033345923925473156,
      "improvement_points": null,
      "runs": [
       "grading:site-jeffan-9b-1211-v1"
      ],
      "notes": "tattrongvu/Jeffan-9B-1211@862d1a7a, its card's Decision Index engine (jeffan_engine:DecisionEngine, transformers 5.17.0, flash-linear-attention 0.5.2, kit decision-index@f8552a1, options max_len and budget 65,536 as its card; without the card's optional causal-conv1d, so its PyTorch reference convolution) behind di_serve.py, our System One wrapper of the engine, BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.05723418541718017,
      "gpu_cost_usd": null
     },
     {
      "name": "d1.1",
      "display_name": "d1.1",
      "tag": "",
      "model_key": "liquid-d1.1",
      "maker": "Liquid AI",
      "kind": "decision",
      "repo": null,
      "commit": null,
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 82.52779120347994,
      "raw": {
       "metric": "accuracy",
       "value": 0.8835186080231996,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.08687460160915417,
      "improvement_points": null,
      "runs": [
       "grading:site-liquid-d1.1-v1"
      ],
      "notes": "Liquid AI decisions API (api.liquid.ai/decisions, model d1.1); one question per request, 8 requests at a time (latency not comparable across different concurrency)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "name": "GLiDE 9B",
      "display_name": "GLiDE 9B",
      "tag": "",
      "model_key": "glide-9b",
      "maker": "Fastino",
      "kind": "decision",
      "repo": "fastino/GLiDE-9B",
      "commit": "ab54df5acba0eebf7df45560ffdaacd64e32df3b",
      "params_total": 7.94,
      "params_active": 7.94,
      "quantization": "BF16",
      "score_0_100": 82.2377960367327,
      "raw": {
       "metric": "accuracy",
       "value": 0.8815853069115515,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.07204617761476283,
      "improvement_points": null,
      "runs": [
       "grading:site-glide-9b-v1"
      ],
      "notes": "fastino/GLiDE-9B@ab54df5a, GLiDE server (python -m glide.server) in vLLM v0.30.0 image with flash-linear-attention 0.5.2, as for GLiDE 2B, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 3 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
      "gpu_hours": 0.05197667056112872,
      "gpu_cost_usd": null
     },
     {
      "name": "Cerno 9B v1.2",
      "display_name": "Cerno 9B v1.2",
      "tag": "",
      "model_key": "cerno-9b-v1.2",
      "maker": "Apertix AI",
      "kind": "decision",
      "repo": "apertixai/cerno-9b-v1.2",
      "commit": "f02a632fbe3a4698cd1ff5c0b929ea240e2b8eda",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 81.87530207829867,
      "raw": {
       "metric": "accuracy",
       "value": 0.8791686805219913,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.028972160463992265,
      "improvement_points": null,
      "runs": [
       "grading:site-cerno-9b-v1.2-v1"
      ],
      "notes": "apertixai/cerno-9b-v1.2@f02a632f, its card's Decision Index engine (CernoEngine, transformers 5.17.0, flash-linear-attention 0.5.2, kit decision-index@9eb2dbe2, the checkpoint's decider_config.json settings, loaded by Hub id and that revision as its card runs it, from the local copy; causal-conv1d not installed, so its PyTorch reference convolution) behind di_serve.py, our System One wrapper of the engine, BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.07257821582541932,
      "gpu_cost_usd": null
     },
     {
      "name": "StartLux-Decision-v1.1-4B-Instant",
      "display_name": "StartLux-Decision-v1.1-4B-Instant",
      "tag": "",
      "model_key": "startlux-decision-v1.1-4b",
      "maker": "StartLux",
      "kind": "decision",
      "repo": "startlux-models/StartLux-Decision-v1.1-4B-Instant",
      "commit": "416088f77cfce09633906a3675c7cd2e5dabcf10",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "BF16",
      "score_0_100": 81.65780570323827,
      "raw": {
       "metric": "accuracy",
       "value": 0.8777187046882552,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.08316936790980808,
      "improvement_points": null,
      "runs": [
       "grading:site-startlux-decision-v1.1-4b-v1"
      ],
      "notes": "startlux-models/StartLux-Decision-v1.1-4B-Instant@416088f7, its package's own server (startlux-decide serve --launch-vllm, StartLux-Decision@a99977d, vLLM 0.31.0) as for its published Reading and Logic & Math runs, BF16, on one DGX Spark (ours), self-hosted; licence CC-BY-NC-4.0 (research use); one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.08060698456742366,
      "gpu_cost_usd": null
     },
     {
      "name": "Dexy-Decision-9B-V2",
      "display_name": "Dexy-Decision-9B-V2",
      "tag": "",
      "model_key": "dexy-decision-9b-v2",
      "maker": "Patrick-18",
      "kind": "decision",
      "repo": "Patrick-18/Dexy-Decision-9B-V2",
      "commit": "4f434e2b5fd9f0addfe3233489571c2dc8ce7a19",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 81.07781536974382,
      "raw": {
       "metric": "accuracy",
       "value": 0.8738521024649589,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.025398751062972964,
      "improvement_points": null,
      "runs": [
       "grading:site-dexy-decision-9b-v2-v1"
      ],
      "notes": "Patrick-18/Dexy-Decision-9B-V2@4f434e2b, its card's Transformers service (d3_server.py, in the environment its requirements.txt pins: torch 2.14.0, transformers 5.19.0), BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.04106403172565293,
      "gpu_cost_usd": null
     },
     {
      "name": "KnowLine-12B-Gen1",
      "display_name": "KnowLine-12B-Gen1",
      "tag": "",
      "model_key": "knowline-12b-gen1",
      "maker": "PelaAI",
      "kind": "decision",
      "repo": "PelaAI/KnowLine-12B-Gen1",
      "commit": "f26fbc2d64092bb2f8caaedbec243172bb3e908f",
      "params_total": 12.0,
      "params_active": 12.0,
      "quantization": "BF16",
      "score_0_100": 80.86031899468341,
      "raw": {
       "metric": "accuracy",
       "value": 0.8724021266312229,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.06837137729635548,
      "improvement_points": null,
      "runs": [
       "grading:site-knowline-12b-gen1-v1"
      ],
      "notes": "PelaAI/KnowLine-12B-Gen1@f26fbc2d, its card's serve_knowline.sh (SGLang, then knowline_server.py, card defaults) in the lmsysorg/sglang image with SGLang 0.5.23 (the card names 0.5.21, which we do not have), BF16, SHA256SUMS checked, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.11182212392749963,
      "gpu_cost_usd": null
     },
     {
      "name": "KnowLine 4B Gen5.5",
      "display_name": "KnowLine 4B Gen5.5",
      "tag": "",
      "model_key": "knowline-4b-gen5.5",
      "maker": "PelaAI",
      "kind": "decision",
      "repo": "PelaAI/KnowLine-4B-Gen5.5",
      "commit": "779944d2cc8ffa19cef386aee83c83bcb967ade2",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "FP8",
      "score_0_100": 80.642822619623,
      "raw": {
       "metric": "accuracy",
       "value": 0.8709521507974867,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.045316565330469655,
      "improvement_points": null,
      "runs": [
       "grading:site-knowline-4b-gen5.5-v1"
      ],
      "notes": "PelaAI/KnowLine-4B-Gen5.5@779944d2 (the site's pin; no longer in the maker's repository, copied from the snapshot its published runs used, SHA256SUMS checked), its card's serve_knowline.sh (SGLang with FP8 at load, then knowline_server.py, card defaults) in the lmsysorg/sglang image with SGLang 0.5.23, on one RTX PRO 6000 (ours), self-hosted; its published runs were on a DGX Spark; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.028165908471719098,
      "gpu_cost_usd": null
     },
     {
      "name": "Adjudex 9B v5",
      "display_name": "Adjudex 9B v5",
      "tag": "",
      "model_key": "adjudex-9b-v5",
      "maker": "newsbang",
      "kind": "decision",
      "repo": "newsbang/adjudex-9b-v5",
      "commit": "ffdcb8254d1b22e40b2bbbf569f143e0b357aeca",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 80.42532624456258,
      "raw": {
       "metric": "accuracy",
       "value": 0.8695021749637506,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.049161628998359276,
      "improvement_points": null,
      "runs": [
       "grading:site-adjudex-9b-v5-v1"
      ],
      "notes": "newsbang/adjudex-9b-v5@ffdcb825, its card's autojev server (the Perplexity Decider v1.1 source, in the environment of its requirements), default settings, BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.04104341335131996,
      "gpu_cost_usd": null
     },
     {
      "name": "MiniCPM5-2B",
      "display_name": "MiniCPM5-2B",
      "tag": "thinking on · chat",
      "model_key": "minicpm5-2b",
      "maker": "OpenBMB",
      "kind": "chat",
      "repo": "openbmb/MiniCPM5-2B",
      "commit": "42433e90dad95123ab87729fd7011e7110629556",
      "params_total": 2.52,
      "params_active": 2.52,
      "quantization": "BF16",
      "score_0_100": 79.2411793136781,
      "raw": {
       "metric": "accuracy",
       "value": 0.8616078620911873,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "grading:chat-minicpm5-2b-seed0-r2-v1",
       "grading:chat-minicpm5-2b-seed1-r2-v1",
       "grading:chat-minicpm5-2b-seed2-r2-v1"
      ],
      "notes": "openbmb/MiniCPM5-2B@42433e90, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 30 requests at a time from the client (latency not comparable across different concurrency); mean of 3 full runs, seeds 0, 1, 2 (min-max across seeds, not a confidence interval); answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; replies without an answer marker count wrong: mean rate 0.06% (per seed in seed_runs; cut off at the token limit per seed: 0, 0, 0); the three runs are fresh runs (14 Nov) with every reply kept, with the same model, sampling settings and requests at a time as the runs they replace and the server settings recorded for them; the earlier runs stay in history; gpu_hours is the mean of the three runs",
      "gpu_hours": 0.11305822975990755,
      "gpu_cost_usd": null,
      "accuracy_basis": "mean of 3 seeds",
      "seed_min": 0.8598356694055099,
      "seed_max": 0.8637022716288062,
      "seed_runs": [
       {
        "run": "grading:chat-minicpm5-2b-seed0-r2-v1",
        "seed": 0,
        "accuracy": 0.8598356694055099,
        "invalid": 3,
        "cutoff": 0,
        "wrong_answers_caught": 250
       },
       {
        "run": "grading:chat-minicpm5-2b-seed1-r2-v1",
        "seed": 1,
        "accuracy": 0.861285645239246,
        "invalid": 0,
        "cutoff": 0,
        "wrong_answers_caught": 258
       },
       {
        "run": "grading:chat-minicpm5-2b-seed2-r2-v1",
        "seed": 2,
        "accuracy": 0.8637022716288062,
        "invalid": 1,
        "cutoff": 0,
        "wrong_answers_caught": 256
       }
      ],
      "invalid_rate_mean": 0.000644433703882713
     },
     {
      "name": "d1",
      "display_name": "d1",
      "tag": "",
      "model_key": "liquid-d1",
      "maker": "Liquid AI",
      "kind": "decision",
      "repo": null,
      "commit": null,
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 79.19284678588689,
      "raw": {
       "metric": "accuracy",
       "value": 0.861285645239246,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.03076448878623012,
      "improvement_points": null,
      "runs": [
       "grading:site-liquid-d1-v1"
      ],
      "notes": "Liquid AI decisions API (api.liquid.ai/decisions, model d1); billed 'd1' route, the site's 'd1:free' was withdrawn (HTTP 404, 13 Nov); one question per request, 8 requests at a time (latency not comparable across different concurrency)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "name": "Perplexity Decider Flash v1.3",
      "display_name": "Perplexity Decider Flash v1.3",
      "tag": "",
      "model_key": "perplexity-decider-flash-v1.3",
      "maker": "Perplexity",
      "kind": "decision",
      "repo": "perplexity-ai/pplx-decider-flash-v1.3",
      "commit": "d06fb3f2d97e0a52325a0d463c450cf0338f44fe",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "BF16",
      "score_0_100": 78.83035282745286,
      "raw": {
       "metric": "accuracy",
       "value": 0.8588690188496858,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.041819332198274745,
      "improvement_points": null,
      "runs": [
       "grading:site-perplexity-decider-flash-v1.3-v1"
      ],
      "notes": "perplexity-ai/pplx-decider-flash-v1.3@d06fb3f2, the runner's pplx_decider_server.py with the checkpoint's own pplx_decider package (as for its published v1.2 runs), BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 3 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
      "gpu_hours": 0.035180532006086604,
      "gpu_cost_usd": null
     },
     {
      "name": "StartLux-Decision-v1.1-9B-Instant",
      "display_name": "StartLux-Decision-v1.1-9B-Instant",
      "tag": "",
      "model_key": "startlux-decision-v1.1-9b",
      "maker": "StartLux",
      "kind": "decision",
      "repo": "startlux-models/StartLux-Decision-v1.1-9B-Instant",
      "commit": "18a17f734966b213b2f00fa9c915e8b9af2a9b20",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 78.83035282745286,
      "raw": {
       "metric": "accuracy",
       "value": 0.8588690188496858,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.0823722971203353,
      "improvement_points": null,
      "runs": [
       "grading:site-startlux-decision-v1.1-9b-v1"
      ],
      "notes": "startlux-models/StartLux-Decision-v1.1-9B-Instant@18a17f73, its package's own server (startlux-decide serve --launch-vllm, StartLux-Decision@a99977d, vLLM 0.31.0), BF16, on one DGX Spark (ours), self-hosted; licence CC-BY-NC-4.0 (research use); one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.15375955299852695,
      "gpu_cost_usd": null
     },
     {
      "name": "Chinese Decision Pointer 4B",
      "display_name": "Chinese Decision Pointer 4B",
      "tag": "",
      "model_key": "chinese-decision-pointer-4b",
      "maker": "quincyqiang",
      "kind": "decision",
      "repo": "quincyqiang/chinese-decision-pointer-4b",
      "commit": "889f23e2655e5cba1208bb17f90526092813f5f2",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "BF16",
      "score_0_100": 78.10536491058483,
      "raw": {
       "metric": "accuracy",
       "value": 0.8540357660705655,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.0666280824070461,
      "improvement_points": null,
      "runs": [
       "grading:site-chinese-decision-pointer-4b-v1"
      ],
      "notes": "quincyqiang/chinese-decision-pointer-4b@889f23e2, its card's Decision Index engine (yanqiangmiffy/chinese-decision@c16450b, ChineseDecisionEngine with debias 'reverse' as its card's run; transformers 5.17.0, kit decision-index@9eb2dbe2, as the card names no kit commit; causal-conv1d not installed, so its PyTorch reference convolution) behind di_serve.py, our System One wrapper of the engine, BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.04414093524818924,
      "gpu_cost_usd": null
     },
     {
      "name": "Perplexity Decider Flash v1.2",
      "display_name": "Perplexity Decider Flash v1.2",
      "tag": "",
      "model_key": "perplexity-decider-flash-v1.2",
      "maker": "Perplexity",
      "kind": "decision",
      "repo": "perplexity-ai/pplx-decider-flash-v1.2",
      "commit": "6d771ad7b5ecff5e93b8fc7b90d4c5ccc341954d",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "BF16",
      "score_0_100": 77.16288061865636,
      "raw": {
       "metric": "accuracy",
       "value": 0.8477525374577091,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.02793073152029759,
      "improvement_points": null,
      "runs": [
       "grading:site-perplexity-decider-flash-v1.2-v1"
      ],
      "notes": "perplexity-ai/pplx-decider-flash-v1.2@6d771ad7, the runner's pplx_decider_server.py with the checkpoint's own pplx_decider package (as for its published v1.2 runs), BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.037100849536345196,
      "gpu_cost_usd": null
     },
     {
      "name": "Gemma 4 E4B",
      "display_name": "Gemma 4 E4B",
      "tag": "thinking on · chat",
      "model_key": "gemma-4-e4b-think",
      "maker": "Google",
      "kind": "chat",
      "repo": "google/gemma-4-E4B-it",
      "commit": "4ecc2ba63bcac23ba4a767fc87025b1450f84c9c",
      "params_total": 8.0,
      "params_active": 8.0,
      "quantization": "BF16",
      "score_0_100": 77.16288061865636,
      "raw": {
       "metric": "accuracy",
       "value": 0.8477525374577091,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "grading:chat-gemma-4-e4b-think-v1"
      ],
      "notes": "google/gemma-4-E4B-it@4ecc2ba6, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 30 requests at a time (latency not comparable across different concurrency); thinking on; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 2 of 2069 replies (0.10%) had no answer marker and count wrong (0 of them cut off at the 32,768-token limit); every reply kept; an earlier attempt of this same run stopped part-way (510 recorded pairs and an unknown number of further requests possibly answered but not recorded) and is kept apart, not merged; this attempt started again from the first pair, so its outcomes are this attempt's recorded outputs, not guaranteed to be the model's first outputs on those pairs",
      "gpu_hours": 0.18707697313942698,
      "gpu_cost_usd": null
     },
     {
      "name": "Gemini Decision 1",
      "display_name": "Gemini Decision 1",
      "tag": "",
      "model_key": "gemini-decision-1",
      "maker": "Google",
      "kind": "decision",
      "repo": null,
      "commit": null,
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 77.01788303528274,
      "raw": {
       "metric": "accuracy",
       "value": 0.8467858869018849,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.03889325895040145,
      "improvement_points": null,
      "runs": [
       "grading:gemini-v1"
      ],
      "notes": "Google Decisions API (generativelanguage.googleapis.com, model gemini-decision-1); one question per request, 8 requests at a time (latency not comparable across different concurrency)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "name": "Gemma 4 12B Unified",
      "display_name": "Gemma 4 12B Unified",
      "tag": "thinking off · chat",
      "model_key": "gemma-4-12b",
      "maker": "Google",
      "kind": "chat",
      "repo": "google/gemma-4-12B-it",
      "commit": "43c9d014b231474fa7212dbb48a762566c2b255d",
      "params_total": 11.95,
      "params_active": 11.95,
      "quantization": "BF16",
      "score_0_100": 77.01788303528274,
      "raw": {
       "metric": "accuracy",
       "value": 0.8467858869018849,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "grading:chat-gemma-4-12b-r2-v1"
      ],
      "notes": "google/gemma-4-12B-it@43c9d014, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time (latency not comparable across different concurrency); thinking off; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 0 of 2069 replies (0%) had no answer marker and count wrong; a fresh run (14-15 Nov) with every reply kept, with the same model, sampling settings and requests at a time as the run it replaces and a server from the same launcher (the earlier run's own server flags were not recorded); the earlier run stays in history",
      "gpu_hours": 0.06945622949589676,
      "gpu_cost_usd": null
     },
     {
      "name": "Elephant-1",
      "display_name": "Elephant-1",
      "tag": "",
      "model_key": "elephant-1",
      "maker": "Base Labs",
      "kind": "decision",
      "repo": "baselabs/elephant-1",
      "commit": "b3f8a93462fb1051a9de031e94a7ff380e254072",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 76.87288545190913,
      "raw": {
       "metric": "accuracy",
       "value": 0.8458192363460609,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.03695449589173516,
      "improvement_points": null,
      "runs": [
       "grading:site-elephant-1-v1"
      ],
      "notes": "baselabs/elephant-1@b3f8a934, the runner's baselabs_server.py in the card's setup.sh environment (OneJev 81ce62f1 with the checkpoint's serve.patch), on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 3 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
      "gpu_hours": 0.056309502522205596,
      "gpu_cost_usd": null
     },
     {
      "name": "Winnow-12B",
      "display_name": "Winnow-12B",
      "tag": "",
      "model_key": "winnow-12b",
      "maker": "EldanRing",
      "kind": "decision",
      "repo": "EldanRing/Winnow-12B",
      "commit": "b2b14213dfa252e6d6b543c8b334762e51772d29",
      "params_total": 12.0,
      "params_active": 12.0,
      "quantization": "Q8_0",
      "score_0_100": 76.58289028516192,
      "raw": {
       "metric": "accuracy",
       "value": 0.8438859352344128,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.08255692633693724,
      "improvement_points": null,
      "runs": [
       "grading:site-winnow-12b-v1"
      ],
      "notes": "EldanRing/Winnow-12B@b2b14213, its Q8_0 GGUF served by winnow-inference@d4631fb, the server its Logic & Math runs use, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.07591675669531456,
      "gpu_cost_usd": null
     },
     {
      "name": "KnowLine-9B-Gen1",
      "display_name": "KnowLine-9B-Gen1",
      "tag": "",
      "model_key": "knowline-9b-gen1",
      "maker": "PelaAI",
      "kind": "decision",
      "repo": "PelaAI/KnowLine-9B-Gen1",
      "commit": "325378d128e9f5eb5088196668f5874c5682b6c0",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 76.14789753504107,
      "raw": {
       "metric": "accuracy",
       "value": 0.8409859835669405,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.035704499480152495,
      "improvement_points": null,
      "runs": [
       "grading:site-knowline-9b-gen1-v1"
      ],
      "notes": "PelaAI/KnowLine-9B-Gen1@325378d1, its card's serve_knowline.sh (SGLang, then knowline_server.py, card defaults) in the lmsysorg/sglang image with SGLang 0.5.23 (the card's release launcher names 0.5.21, which we do not have), BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.06016774425950634,
      "gpu_cost_usd": null
     },
     {
      "name": "Granite-4.2-3B",
      "display_name": "Granite-4.2-3B",
      "tag": "thinking on · chat",
      "model_key": "granite-4.2-3b-think",
      "maker": "IBM",
      "kind": "chat",
      "repo": "ibm-granite/granite-4.2-3b",
      "commit": "4e9c275e232f10211ec88644d106a1f6a1b60e06",
      "params_total": 3.0,
      "params_active": 3.0,
      "quantization": "BF16",
      "score_0_100": 75.85790236829386,
      "raw": {
       "metric": "accuracy",
       "value": 0.8390526824552924,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "grading:chat-granite-4.2-3b-think-v1"
      ],
      "notes": "ibm-granite/granite-4.2-3b@4e9c275e, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 30 requests at a time (latency not comparable across different concurrency); thinking on; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 0 of 2069 replies (0%) had no answer marker and count wrong; every reply kept",
      "gpu_hours": 0.603654108024119,
      "gpu_cost_usd": null
     },
     {
      "name": "LFM3-1.7B",
      "display_name": "LFM3-1.7B",
      "tag": "thinking on · chat",
      "model_key": "lfm3-1.7b",
      "maker": "Liquid AI",
      "kind": "chat",
      "repo": "LiquidAI/LFM3-1.7B",
      "commit": "0dc4268b561499c648c7ba7c9466bf3d8d01517a",
      "params_total": 1.73,
      "params_active": 1.73,
      "quantization": "BF16",
      "score_0_100": 75.78540357660704,
      "raw": {
       "metric": "accuracy",
       "value": 0.8385693571773803,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "grading:chat-lfm3-1.7b-seed0-replay-v1",
       "grading:chat-lfm3-1.7b-seed1-r2-v1",
       "grading:chat-lfm3-1.7b-seed2-r2-v1"
      ],
      "notes": "LiquidAI/LFM3-1.7B@0dc4268b, BF16, cu135-nightly-be47189f37f6162991366ff6b91539f6e9d1f432 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 30 requests at a time from the client (latency not comparable across different concurrency); mean of 3 full runs, seeds 0, 1, 2 (min-max across seeds, not a confidence interval); answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; replies without an answer marker count wrong: mean rate 0.06% (per seed in seed_runs; cut off at the token limit per seed: 2, 1, 1); the three runs are fresh runs (14 Nov) with every reply kept, with the same model, sampling settings and requests at a time as the runs they replace and the server settings recorded for them; seed 0 is the rerun made to diagnose the missing answer lines; the earlier runs stay in history; gpu_hours is the mean of the three runs",
      "gpu_hours": 0.15417772855343545,
      "gpu_cost_usd": null,
      "accuracy_basis": "mean of 3 seeds",
      "seed_min": 0.832769453842436,
      "seed_max": 0.8434026099565007,
      "seed_runs": [
       {
        "run": "grading:chat-lfm3-1.7b-seed0-replay-v1",
        "seed": 0,
        "accuracy": 0.8395360077332045,
        "invalid": 2,
        "cutoff": 2,
        "wrong_answers_caught": 215
       },
       {
        "run": "grading:chat-lfm3-1.7b-seed1-r2-v1",
        "seed": 1,
        "accuracy": 0.832769453842436,
        "invalid": 1,
        "cutoff": 1,
        "wrong_answers_caught": 194
       },
       {
        "run": "grading:chat-lfm3-1.7b-seed2-r2-v1",
        "seed": 2,
        "accuracy": 0.8434026099565007,
        "invalid": 1,
        "cutoff": 1,
        "wrong_answers_caught": 216
       }
      ],
      "invalid_rate_mean": 0.000644433703882713
     },
     {
      "name": "Verdin-4B-v1.1",
      "display_name": "Verdin-4B-v1.1",
      "tag": "",
      "model_key": "verdin-4b-v1.1",
      "maker": "victorbennett",
      "kind": "decision",
      "repo": "victorbennett/Verdin-4B-v1.1",
      "commit": "45830d216bb7e45f43f0a482c131a7cafa6d189f",
      "params_total": 4.0,
      "params_active": 4.0,
      "quantization": "BF16",
      "score_0_100": 75.35041082648621,
      "raw": {
       "metric": "accuracy",
       "value": 0.8356694055099082,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.03399785843308816,
      "improvement_points": null,
      "runs": [
       "grading:site-verdin-4b-v1.1-v1"
      ],
      "notes": "victorbennett/Verdin-4B-v1.1@45830d21, its card's own server (python -m verdin.server, in the package versions its card lists: torch 2.8.0 cu128, transformers 5.17.0, flash-linear-attention 0.5.2, triton 3.8.0; no causal-conv1d, so its own torch conv path), BF16, on one RTX PRO 6000 (ours), self-hosted; licence CC-BY-NC-4.0 (research use); one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.031078735832046005,
      "gpu_cost_usd": null
     },
     {
      "name": "KnowLine-9B-Gen2",
      "display_name": "KnowLine-9B-Gen2",
      "tag": "",
      "model_key": "knowline-9b-gen2",
      "maker": "PelaAI",
      "kind": "decision",
      "repo": "PelaAI/KnowLine-9B-Gen2",
      "commit": "8ec8eb609bc10aa6050ae3754aee419d458fae42",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 75.27791203479943,
      "raw": {
       "metric": "accuracy",
       "value": 0.8351860802319961,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.04495010483366471,
      "improvement_points": null,
      "runs": [
       "grading:site-knowline-9b-gen2-v1"
      ],
      "notes": "PelaAI/KnowLine-9B-Gen2@8ec8eb60, its card's serve_knowline.sh (SGLang, then knowline_server.py, card defaults) in the lmsysorg/sglang image with SGLang 0.5.23 (the card names 0.5.21, which we do not have), BF16, SHA256SUMS checked, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.07386159999956078,
      "gpu_cost_usd": null
     },
     {
      "name": "Gemma 4 E4B",
      "display_name": "Gemma 4 E4B",
      "tag": "thinking on · sampled (temperature 1.0) · chat",
      "model_key": "gemma-4-e4b-think-sampled",
      "maker": "Google",
      "kind": "chat",
      "repo": "google/gemma-4-E4B-it",
      "commit": "4ecc2ba63bcac23ba4a767fc87025b1450f84c9c",
      "params_total": 8.0,
      "params_active": 8.0,
      "quantization": "BF16",
      "score_0_100": 75.1087481875302,
      "raw": {
       "metric": "accuracy",
       "value": 0.8340583212502014,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "grading:chat-gemma-4-e4b-think-sampled-seed0-v1",
       "grading:chat-gemma-4-e4b-think-sampled-seed1-v1",
       "grading:chat-gemma-4-e4b-think-sampled-seed2-v1"
      ],
      "notes": "google/gemma-4-E4B-it@4ecc2ba6, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 30 requests at a time from the client (latency not comparable across different concurrency); mean of 3 full runs, seeds 0, 1, 2 (min-max across seeds, not a confidence interval); answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; replies without an answer marker count wrong: mean rate 2% (per seed in seed_runs; cut off at the token limit per seed: 0, 0, 0); three new runs (16 Nov, every reply kept), seeds 0, 1, 2 at Gemma's own sampling (its generation_config.json: temperature 1.0, top_p 0.95, top_k 64; Ümit #10554), with the weights, template and token limit of the temperature-0 row, which stays as its own row; their server settings recorded; gpu_hours is the mean of the three runs",
      "gpu_hours": 0.13427235311653798,
      "gpu_cost_usd": null,
      "accuracy_basis": "mean of 3 seeds",
      "seed_min": 0.8274528757854036,
      "seed_max": 0.8395360077332045,
      "seed_runs": [
       {
        "run": "grading:chat-gemma-4-e4b-think-sampled-seed0-v1",
        "seed": 0,
        "accuracy": 0.8395360077332045,
        "invalid": 34,
        "cutoff": 0,
        "wrong_answers_caught": 216
       },
       {
        "run": "grading:chat-gemma-4-e4b-think-sampled-seed1-v1",
        "seed": 1,
        "accuracy": 0.8274528757854036,
        "invalid": 31,
        "cutoff": 0,
        "wrong_answers_caught": 208
       },
       {
        "run": "grading:chat-gemma-4-e4b-think-sampled-seed2-v1",
        "seed": 2,
        "accuracy": 0.8351860802319961,
        "invalid": 30,
        "cutoff": 0,
        "wrong_answers_caught": 211
       }
      ],
      "invalid_rate_mean": 0.015305300467214436
     },
     {
      "name": "LFM3-1.7B",
      "display_name": "LFM3-1.7B",
      "tag": "thinking off · chat",
      "model_key": "lfm3-1.7b-nothink",
      "maker": "Liquid AI",
      "kind": "chat",
      "repo": "LiquidAI/LFM3-1.7B",
      "commit": "0dc4268b561499c648c7ba7c9466bf3d8d01517a",
      "params_total": 1.73,
      "params_active": 1.73,
      "quantization": "BF16",
      "score_0_100": 74.89125181246979,
      "raw": {
       "metric": "accuracy",
       "value": 0.8326083454164653,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "grading:chat-lfm3-1.7b-nothink-r2-v1",
       "grading:chat-lfm3-1.7b-nothink-seed1-r2-v1",
       "grading:chat-lfm3-1.7b-nothink-seed2-r2-v1"
      ],
      "notes": "LiquidAI/LFM3-1.7B@0dc4268b, BF16, cu135-nightly-be47189f37f6162991366ff6b91539f6e9d1f432 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time from the client (latency not comparable across different concurrency); mean of 3 full runs, seeds 0, 1, 2 (min-max across seeds, not a confidence interval); answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; replies without an answer marker count wrong: mean rate 0.08% (per seed in seed_runs; cut off at the token limit per seed: 2, 2, 1); seed 0 is the published single run (14 Nov, every reply kept); seeds 1 and 2 are new runs (15 Nov, every reply kept) with the same model, sampling settings, requests at a time and server launcher, only the seed changed, their server settings recorded; gpu_hours is the mean of the three runs",
      "gpu_hours": 0.10339265255892785,
      "gpu_cost_usd": null,
      "accuracy_basis": "mean of 3 seeds",
      "seed_min": 0.8231029482841953,
      "seed_max": 0.8376027066215563,
      "seed_runs": [
       {
        "run": "grading:chat-lfm3-1.7b-nothink-r2-v1",
        "seed": 0,
        "accuracy": 0.8376027066215563,
        "invalid": 2,
        "cutoff": 2,
        "wrong_answers_caught": 212
       },
       {
        "run": "grading:chat-lfm3-1.7b-nothink-seed1-r2-v1",
        "seed": 1,
        "accuracy": 0.8371193813436443,
        "invalid": 2,
        "cutoff": 2,
        "wrong_answers_caught": 209
       },
       {
        "run": "grading:chat-lfm3-1.7b-nothink-seed2-r2-v1",
        "seed": 2,
        "accuracy": 0.8231029482841953,
        "invalid": 1,
        "cutoff": 1,
        "wrong_answers_caught": 197
       }
      ],
      "invalid_rate_mean": 0.0008055421298533914
     },
     {
      "name": "Valen-Decision-1.5-9B",
      "display_name": "Valen-Decision-1.5-9B",
      "tag": "",
      "model_key": "valen-decision-1.5-9b",
      "maker": "Valen-Team",
      "kind": "decision",
      "repo": "Valen-Team/Valen-Decision-1.5-9B",
      "commit": "5a512117ebc0f2b69f311becfcf47cdef8abf568",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 74.26292895118414,
      "raw": {
       "metric": "accuracy",
       "value": 0.8284195263412276,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.03150234591802053,
      "improvement_points": null,
      "runs": [
       "grading:site-valen-decision-1.5-9b-v1"
      ],
      "notes": "Valen-Team/Valen-Decision-1.5-9B@5a512117, its card's vLLM deployment (valen-vllm, the Valen plugin at Valen-Team/Valen@773d3a64 in the official vllm/vllm-openai:v0.33.0 image), BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.04507349295907858,
      "gpu_cost_usd": null
     },
     {
      "name": "Cygnet",
      "display_name": "Cygnet",
      "tag": "",
      "model_key": "cygnet",
      "maker": "Blockbrain Labs",
      "kind": "decision",
      "repo": null,
      "commit": null,
      "params_total": 11.95,
      "params_active": 11.95,
      "quantization": "BF16",
      "score_0_100": 72.88545190913484,
      "raw": {
       "metric": "accuracy",
       "value": 0.819236346060899,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.06509097176030178,
      "improvement_points": null,
      "runs": [
       "grading:site-cygnet-v1"
      ],
      "notes": "no weights of its own: google/gemma-4-12B-it@707f0a3b, unchanged, on vLLM 0.30.0 behind cygnet-recipe's shim (blockbrain-ai/cygnet-recipe@3cf591c, temperature 3.4), the server its Logic & Math runs use, BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.05341540666628283,
      "gpu_cost_usd": null
     },
     {
      "name": "Gemma 4 E2B",
      "display_name": "Gemma 4 E2B",
      "tag": "thinking on · chat",
      "model_key": "gemma-4-e2b-think",
      "maker": "Google",
      "kind": "chat",
      "repo": "google/gemma-4-E2B-it",
      "commit": "4adf15748de9e1a0097b6274cee9c15836c41c37",
      "params_total": 5.1,
      "params_active": 5.1,
      "quantization": "BF16",
      "score_0_100": 71.14548090865152,
      "raw": {
       "metric": "accuracy",
       "value": 0.8076365393910101,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "grading:chat-gemma-4-e2b-think-r2-v1"
      ],
      "notes": "google/gemma-4-E2B-it@4adf1574, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time (latency not comparable across different concurrency); thinking on; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 0 of 2069 replies (0%) had no answer marker and count wrong; a fresh run (14-15 Nov) with every reply kept, with the same model, sampling settings and requests at a time as the run it replaces and a server from the same launcher (the earlier run's own server flags were not recorded); the earlier run stays in history",
      "gpu_hours": 0.039903491282575095,
      "gpu_cost_usd": null
     },
     {
      "name": "Qwen3.5-2B",
      "display_name": "Qwen3.5-2B",
      "tag": "thinking on · chat",
      "model_key": "qwen3.5-2b",
      "maker": "Alibaba",
      "kind": "chat",
      "repo": "Qwen/Qwen3.5-2B",
      "commit": "15852e8c16360a2fea060d615a32b45270f8a8fc",
      "params_total": 2.27,
      "params_active": 2.27,
      "quantization": "BF16",
      "score_0_100": 70.34799420009666,
      "raw": {
       "metric": "accuracy",
       "value": 0.8023199613339778,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "grading:chat-qwen3.5-2b-r2-v1"
      ],
      "notes": "Qwen/Qwen3.5-2B@15852e8c, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time (latency not comparable across different concurrency); thinking on; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 78 of 2069 replies (4%) had no answer marker and count wrong (5 of them cut off at the 32,768-token limit); a fresh run (14-15 Nov) with every reply kept, with the same model, sampling settings and requests at a time as the run it replaces and a server from the same launcher (the earlier run's own server flags were not recorded); the earlier run stays in history; an earlier attempt of this same run stopped part-way (1,210 recorded pairs and 64 further requests possibly answered but not recorded) and is kept apart, not merged; this attempt started again from the first pair, so its outcomes are this attempt's recorded outputs, not guaranteed to be the model's first outputs on those pairs",
      "gpu_hours": 0.3654148079916184,
      "gpu_cost_usd": null
     },
     {
      "name": "Clef-Flash",
      "display_name": "Clef-Flash",
      "tag": "",
      "model_key": "clef-flash",
      "maker": "Cloudflare",
      "kind": "decision",
      "repo": "Cloudflare/clef-flash",
      "commit": "fde727a287004204b7518dcc983fe64379776712",
      "params_total": 9.0,
      "params_active": 9.0,
      "quantization": "BF16",
      "score_0_100": 69.76800386660223,
      "raw": {
       "metric": "accuracy",
       "value": 0.7984533591106815,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.05931271145480908,
      "improvement_points": null,
      "runs": [
       "grading:site-clef-flash-v1"
      ],
      "notes": "Cloudflare/clef-flash@fde727a2, the server its Logic & Math runs use, BF16, on one RTX PRO 6000 (ours), self-hosted; server probabilities are rounded to four decimal places and retained without renormalization; ECE uses the reported maximum probability; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.08946663268202902,
      "gpu_cost_usd": null
     },
     {
      "name": "Gemma 4 12B Unified",
      "display_name": "Gemma 4 12B Unified",
      "tag": "thinking on · chat",
      "model_key": "gemma-4-12b-think",
      "maker": "Google",
      "kind": "chat",
      "repo": "google/gemma-4-12B-it",
      "commit": "43c9d014b231474fa7212dbb48a762566c2b255d",
      "params_total": 11.95,
      "params_active": 11.95,
      "quantization": "BF16",
      "score_0_100": 67.66553890768488,
      "raw": {
       "metric": "accuracy",
       "value": 0.7844369260512325,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "grading:chat-gemma-4-12b-think-r2-v1"
      ],
      "notes": "google/gemma-4-12B-it@43c9d014, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time (latency not comparable across different concurrency); thinking on; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 296 of 2069 replies (14%) had no answer marker and count wrong (295 of them cut off at the 32,768-token limit); a fresh run (14-15 Nov) with every reply kept, with the same model, sampling settings and requests at a time as the run it replaces and a server from the same launcher (the earlier run's own server flags were not recorded); the earlier run stays in history",
      "gpu_hours": 2.486874496019143,
      "gpu_cost_usd": null
     },
     {
      "name": "MiniCPM5-2B",
      "display_name": "MiniCPM5-2B",
      "tag": "thinking off · chat",
      "model_key": "minicpm5-2b-nothink",
      "maker": "OpenBMB",
      "kind": "chat",
      "repo": "openbmb/MiniCPM5-2B",
      "commit": "42433e90dad95123ab87729fd7011e7110629556",
      "params_total": 2.52,
      "params_active": 2.52,
      "quantization": "BF16",
      "score_0_100": 66.988883518608,
      "raw": {
       "metric": "accuracy",
       "value": 0.7799258901240534,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "grading:chat-minicpm5-2b-nothink-r2-v1",
       "grading:chat-minicpm5-2b-nothink-seed1-r2-v1",
       "grading:chat-minicpm5-2b-nothink-seed2-r2-v1"
      ],
      "notes": "openbmb/MiniCPM5-2B@42433e90, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time from the client (latency not comparable across different concurrency); mean of 3 full runs, seeds 0, 1, 2 (min-max across seeds, not a confidence interval); answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; replies without an answer marker count wrong: mean rate 0.24% (per seed in seed_runs; cut off at the token limit per seed: 1, 0, 0); seed 0 is the published single run (14 Nov, every reply kept); seeds 1 and 2 are new runs (15 Nov, every reply kept) with the same model, sampling settings, requests at a time and server launcher, only the seed changed, their server settings recorded; gpu_hours is the mean of the three runs (GPU seconds per seed: 219, 76, 76, at the same recorded settings; the cause of the difference is not known)",
      "gpu_hours": 0.02949326808467177,
      "gpu_cost_usd": null,
      "accuracy_basis": "mean of 3 seeds",
      "seed_min": 0.7796036732721121,
      "seed_max": 0.7800869985500242,
      "seed_runs": [
       {
        "run": "grading:chat-minicpm5-2b-nothink-r2-v1",
        "seed": 0,
        "accuracy": 0.7800869985500242,
        "invalid": 6,
        "cutoff": 1,
        "wrong_answers_caught": 93
       },
       {
        "run": "grading:chat-minicpm5-2b-nothink-seed1-r2-v1",
        "seed": 1,
        "accuracy": 0.7800869985500242,
        "invalid": 5,
        "cutoff": 0,
        "wrong_answers_caught": 94
       },
       {
        "run": "grading:chat-minicpm5-2b-nothink-seed2-r2-v1",
        "seed": 2,
        "accuracy": 0.7796036732721121,
        "invalid": 4,
        "cutoff": 0,
        "wrong_answers_caught": 97
       }
      ],
      "invalid_rate_mean": 0.002416626389560174
     },
     {
      "name": "Qwen3.5-4B",
      "display_name": "Qwen3.5-4B",
      "tag": "thinking on · chat",
      "model_key": "qwen3.5-4b",
      "maker": "Alibaba",
      "kind": "chat",
      "repo": "Qwen/Qwen3.5-4B",
      "commit": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",
      "params_total": 4.66,
      "params_active": 4.66,
      "quantization": "BF16",
      "score_0_100": 66.21556307394877,
      "raw": {
       "metric": "accuracy",
       "value": 0.7747704204929918,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "grading:chat-qwen3.5-4b-r2-v1"
      ],
      "notes": "Qwen/Qwen3.5-4B@851bf6e8, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time (latency not comparable across different concurrency); thinking on; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 179 of 2069 replies (9%) had no answer marker and count wrong (0 of them cut off at the 32,768-token limit); a fresh run (14-15 Nov) with every reply kept, with the same model, sampling settings and requests at a time as the run it replaces and a server from the same launcher (the earlier run's own server flags were not recorded); the earlier run stays in history",
      "gpu_hours": 0.4258198070850944,
      "gpu_cost_usd": null
     },
     {
      "name": "StartLux-Decision-v1.1-2B-Instant",
      "display_name": "StartLux-Decision-v1.1-2B-Instant",
      "tag": "",
      "model_key": "startlux-decision-v1.1-2b",
      "maker": "StartLux",
      "kind": "decision",
      "repo": "startlux-models/StartLux-Decision-v1.1-2B-Instant",
      "commit": "f1e6cd147dda82c9ffa3e8e725d5c1f175566fb1",
      "params_total": 2.0,
      "params_active": 2.0,
      "quantization": "BF16",
      "score_0_100": 65.99806669888835,
      "raw": {
       "metric": "accuracy",
       "value": 0.7733204446592556,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.07203038917648069,
      "improvement_points": null,
      "runs": [
       "grading:site-startlux-decision-v1.1-2b-v1"
      ],
      "notes": "startlux-models/StartLux-Decision-v1.1-2B-Instant@f1e6cd14, the server its Logic & Math runs use, BF16, on one DGX Spark (ours), self-hosted; licence CC-BY-NC-4.0 (research use); one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.04491619354242643,
      "gpu_cost_usd": null
     },
     {
      "name": "GLiDE 2B",
      "display_name": "GLiDE 2B",
      "tag": "",
      "model_key": "glide-2b",
      "maker": "Fastino",
      "kind": "decision",
      "repo": "fastino/GLiDE-2B",
      "commit": "00f122c120cdeed82284106cb77d9bab5a50ad16",
      "params_total": 1.88,
      "params_active": 1.88,
      "quantization": "BF16",
      "score_0_100": 65.63557274045432,
      "raw": {
       "metric": "accuracy",
       "value": 0.7709038182696955,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.03954780247703763,
      "improvement_points": null,
      "runs": [
       "grading:site-glide-2b-v1"
      ],
      "notes": "fastino/GLiDE-2B@00f122c1, GLiDE server (python -m glide.server) in vLLM v0.30.0 image with flash-linear-attention 0.5.2 on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.024994372462938563,
      "gpu_cost_usd": null
     },
     {
      "name": "Qwen3.5-4B",
      "display_name": "Qwen3.5-4B",
      "tag": "thinking off · chat",
      "model_key": "qwen3.5-4b-nothink",
      "maker": "Alibaba",
      "kind": "chat",
      "repo": "Qwen/Qwen3.5-4B",
      "commit": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",
      "params_total": 4.66,
      "params_active": 4.66,
      "quantization": "BF16",
      "score_0_100": 64.62058965683906,
      "raw": {
       "metric": "accuracy",
       "value": 0.764137264378927,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "grading:chat-qwen3.5-4b-nothink-v1",
       "grading:chat-qwen3.5-4b-nothink-seed1-v1",
       "grading:chat-qwen3.5-4b-nothink-seed2-v1"
      ],
      "notes": "Qwen/Qwen3.5-4B@851bf6e8, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time from the client (latency not comparable across different concurrency); mean of 3 full runs, seeds 0, 1, 2 (min-max across seeds, not a confidence interval); answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; replies without an answer marker count wrong: mean rate 0.97% (per seed in seed_runs; cut off at the token limit per seed: 0, 1, 0); seed 0 is the published single run (every reply kept); seeds 1 and 2 are new runs (16 Nov, every reply kept) with the same model, sampling settings, requests at a time and server launcher, only the seed changed, their server settings recorded; gpu_hours is the mean of the three runs",
      "gpu_hours": 0.0852996435253031,
      "gpu_cost_usd": null,
      "accuracy_basis": "mean of 3 seeds",
      "seed_min": 0.7631706138231029,
      "seed_max": 0.7655872402126631,
      "seed_runs": [
       {
        "run": "grading:chat-qwen3.5-4b-nothink-v1",
        "seed": 0,
        "accuracy": 0.763653939101015,
        "invalid": 20,
        "cutoff": 0,
        "wrong_answers_caught": 196
       },
       {
        "run": "grading:chat-qwen3.5-4b-nothink-seed1-v1",
        "seed": 1,
        "accuracy": 0.7631706138231029,
        "invalid": 25,
        "cutoff": 1,
        "wrong_answers_caught": 202
       },
       {
        "run": "grading:chat-qwen3.5-4b-nothink-seed2-v1",
        "seed": 2,
        "accuracy": 0.7655872402126631,
        "invalid": 15,
        "cutoff": 0,
        "wrong_answers_caught": 206
       }
      ],
      "invalid_rate_mean": 0.009666505558240696
     },
     {
      "name": "Metask-Jev-4B",
      "display_name": "Metask-Jev-4B",
      "tag": "",
      "model_key": "metask-jev-4b",
      "maker": "Metask Lab",
      "kind": "decision",
      "repo": "wayfind/metask-jev-4b-policy-mix",
      "commit": "ea20fe85b28733b1522721dec119bec50947a869",
      "params_total": 4.54,
      "params_active": 4.54,
      "quantization": "BF16",
      "score_0_100": 61.720637989366836,
      "raw": {
       "metric": "accuracy",
       "value": 0.7448042532624456,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.06794952198236447,
      "improvement_points": null,
      "runs": [
       "grading:site-metask-jev-4b-v1"
      ],
      "notes": "wayfind/metask-jev-4b-policy-mix@ea20fe85, the server its Logic & Math runs use, BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.07693864276212224,
      "gpu_cost_usd": null
     },
     {
      "name": "KnowLine-2B-Gen1",
      "display_name": "KnowLine-2B-Gen1",
      "tag": "",
      "model_key": "knowline-2b-gen1",
      "maker": "PelaAI",
      "kind": "decision",
      "repo": "PelaAI/KnowLine-2B-Gen1",
      "commit": "53245d470a4ce316e32fd0d18c24e142c697f40b",
      "params_total": 2.0,
      "params_active": 2.0,
      "quantization": "BF16",
      "score_0_100": 61.06814886418559,
      "raw": {
       "metric": "accuracy",
       "value": 0.7404543257612373,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.026506755692241563,
      "improvement_points": null,
      "runs": [
       "grading:site-knowline-2b-gen1-v1"
      ],
      "notes": "PelaAI/KnowLine-2B-Gen1@53245d47, the server its Logic & Math runs use, BF16, on one DGX Spark (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.07461765998593127,
      "gpu_cost_usd": null
     },
     {
      "name": "GLiDE 0.8B",
      "display_name": "GLiDE 0.8B",
      "tag": "",
      "model_key": "glide-0.8b",
      "maker": "Fastino",
      "kind": "decision",
      "repo": "fastino/GLiDE-0.8B",
      "commit": "9eacdeec79fa48ec52a8d6f54b331376b4bbfaf0",
      "params_total": 0.75,
      "params_active": 0.75,
      "quantization": "BF16",
      "score_0_100": 60.77815369743836,
      "raw": {
       "metric": "accuracy",
       "value": 0.7385210246495891,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.03848586578633559,
      "improvement_points": null,
      "runs": [
       "grading:site-glide-0.8b-v1"
      ],
      "notes": "fastino/GLiDE-0.8B@9eacdeec, the server its Logic & Math runs use, BF16, on one DGX Spark (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.036354799436418,
      "gpu_cost_usd": null
     },
     {
      "name": "Strands Decider 2B",
      "display_name": "Strands Decider 2B",
      "tag": "",
      "model_key": "strands-decider-2b",
      "maker": "Strands Agents",
      "kind": "decision",
      "repo": "StrandsAgents/strands-decider-2B-hobson-v21",
      "commit": "2b52a6235c1b8306bbfa30b00b9d4b74b63a39f5",
      "params_total": 2.0,
      "params_active": 2.0,
      "quantization": "BF16",
      "score_0_100": 60.34316094731754,
      "raw": {
       "metric": "accuracy",
       "value": 0.735621072982117,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.04984509424842916,
      "improvement_points": null,
      "runs": [
       "grading:site-strands-decider-2b-v1"
      ],
      "notes": "StrandsAgents/strands-decider-2B-hobson-v21@2b52a623 (a LoRA and head on Qwen3.5-2B-Base), the server its Logic & Math runs use, BF16, on one RTX PRO 6000 (ours), self-hosted; server probabilities are rounded to four decimal places and retained without renormalization; ECE uses the reported maximum probability; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.02474473185662914,
      "gpu_cost_usd": null
     },
     {
      "name": "LFM2.5-8B-A1B",
      "display_name": "LFM2.5-8B-A1B",
      "tag": "thinking on (no off mode) · chat",
      "model_key": "lfm2.5-8b-a1b-think",
      "maker": "Liquid AI",
      "kind": "chat",
      "repo": "LiquidAI/LFM2.5-8B-A1B",
      "commit": "dd83f5cb02aa2938b66bba9a9998d38d0517c1f3",
      "params_total": 8.3,
      "params_active": 1.5,
      "quantization": "BF16",
      "score_0_100": 57.733204446592545,
      "raw": {
       "metric": "accuracy",
       "value": 0.7182213629772837,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "grading:chat-lfm2.5-8b-a1b-think-v1"
      ],
      "notes": "LiquidAI/LFM2.5-8B-A1B@dd83f5cb, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 30 requests at a time (latency not comparable across different concurrency); thinking on; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 0 of 2069 replies (0%) had no answer marker and count wrong; every reply kept; an earlier attempt of this same run stopped part-way (45 recorded pairs and an unknown number of further requests possibly answered but not recorded) and is kept apart, not merged; this attempt started again from the first pair, so its outcomes are this attempt's recorded outputs, not guaranteed to be the model's first outputs on those pairs",
      "gpu_hours": 0.2022492439606659,
      "gpu_cost_usd": null
     },
     {
      "name": "Gemma 4 E2B",
      "display_name": "Gemma 4 E2B",
      "tag": "thinking off · sampled (temperature 1.0) · chat",
      "model_key": "gemma-4-e2b-sampled",
      "maker": "Google",
      "kind": "chat",
      "repo": "google/gemma-4-E2B-it",
      "commit": "4adf15748de9e1a0097b6274cee9c15836c41c37",
      "params_total": 5.1,
      "params_active": 5.1,
      "quantization": "BF16",
      "score_0_100": 55.171580473658764,
      "raw": {
       "metric": "accuracy",
       "value": 0.7011438698243918,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "grading:chat-gemma-4-e2b-sampled-seed0-v1",
       "grading:chat-gemma-4-e2b-sampled-seed1-v1",
       "grading:chat-gemma-4-e2b-sampled-seed2-v1"
      ],
      "notes": "google/gemma-4-E2B-it@4adf1574, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time from the client (latency not comparable across different concurrency); mean of 3 full runs, seeds 0, 1, 2 (min-max across seeds, not a confidence interval); answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; replies without an answer marker count wrong: mean rate 0.03% (per seed in seed_runs; cut off at the token limit per seed: 0, 0, 0); three new runs (16 Nov, every reply kept), seeds 0, 1, 2 at Gemma's own sampling (its generation_config.json: temperature 1.0, top_p 0.95, top_k 64; Ümit #10554), with the weights, template and token limit of the temperature-0 row, which stays as its own row; their server settings recorded; gpu_hours is the mean of the three runs",
      "gpu_hours": 0.018991241379863,
      "gpu_cost_usd": null,
      "accuracy_basis": "mean of 3 seeds",
      "seed_min": 0.6969550507491542,
      "seed_max": 0.7056549057515709,
      "seed_runs": [
       {
        "run": "grading:chat-gemma-4-e2b-sampled-seed0-v1",
        "seed": 0,
        "accuracy": 0.6969550507491542,
        "invalid": 1,
        "cutoff": 0,
        "wrong_answers_caught": 22
       },
       {
        "run": "grading:chat-gemma-4-e2b-sampled-seed1-v1",
        "seed": 1,
        "accuracy": 0.7056549057515709,
        "invalid": 0,
        "cutoff": 0,
        "wrong_answers_caught": 29
       },
       {
        "run": "grading:chat-gemma-4-e2b-sampled-seed2-v1",
        "seed": 2,
        "accuracy": 0.7008216529724505,
        "invalid": 1,
        "cutoff": 0,
        "wrong_answers_caught": 25
       }
      ],
      "invalid_rate_mean": 0.0003222168519413565
     },
     {
      "name": "Gemma 4 E2B",
      "display_name": "Gemma 4 E2B",
      "tag": "thinking off · chat",
      "model_key": "gemma-4-e2b",
      "maker": "Google",
      "kind": "chat",
      "repo": "google/gemma-4-E2B-it",
      "commit": "4adf15748de9e1a0097b6274cee9c15836c41c37",
      "params_total": 5.1,
      "params_active": 5.1,
      "quantization": "BF16",
      "score_0_100": 54.76075398743354,
      "raw": {
       "metric": "accuracy",
       "value": 0.6984050265828903,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "grading:chat-gemma-4-e2b-r2-v1"
      ],
      "notes": "google/gemma-4-E2B-it@4adf1574, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time (latency not comparable across different concurrency); thinking off; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 0 of 2069 replies (0%) had no answer marker and count wrong; a fresh run (14-15 Nov) with every reply kept, with the same model, sampling settings and requests at a time as the run it replaces and a server from the same launcher (the earlier run's own server flags were not recorded); the earlier run stays in history",
      "gpu_hours": 0.010497344808671188,
      "gpu_cost_usd": null
     },
     {
      "name": "Qwen3.5-2B",
      "display_name": "Qwen3.5-2B",
      "tag": "thinking off · chat",
      "model_key": "qwen3.5-2b-nothink",
      "maker": "Alibaba",
      "kind": "chat",
      "repo": "Qwen/Qwen3.5-2B",
      "commit": "15852e8c16360a2fea060d615a32b45270f8a8fc",
      "params_total": 2.0,
      "params_active": 2.0,
      "quantization": "BF16",
      "score_0_100": 50.24166263895601,
      "raw": {
       "metric": "accuracy",
       "value": 0.6682777509263734,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "grading:chat-qwen3.5-2b-nothink-v1",
       "grading:chat-qwen3.5-2b-nothink-seed1-v1",
       "grading:chat-qwen3.5-2b-nothink-seed2-v1"
      ],
      "notes": "Qwen/Qwen3.5-2B@15852e8c, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time from the client (latency not comparable across different concurrency); mean of 3 full runs, seeds 0, 1, 2 (min-max across seeds, not a confidence interval); answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; replies without an answer marker count wrong: mean rate 2% (per seed in seed_runs; cut off at the token limit per seed: 1, 0, 0); seed 0 is the published single run (every reply kept); seeds 1 and 2 are new runs (16 Nov, every reply kept) with the same model, sampling settings, requests at a time and server launcher, only the seed changed, their server settings recorded; gpu_hours is the mean of the three runs",
      "gpu_hours": 0.052665647040142136,
      "gpu_cost_usd": null,
      "accuracy_basis": "mean of 3 seeds",
      "seed_min": 0.6621556307394877,
      "seed_max": 0.6718221362977284,
      "seed_runs": [
       {
        "run": "grading:chat-qwen3.5-2b-nothink-v1",
        "seed": 0,
        "accuracy": 0.6621556307394877,
        "invalid": 37,
        "cutoff": 1,
        "wrong_answers_caught": 108
       },
       {
        "run": "grading:chat-qwen3.5-2b-nothink-seed1-v1",
        "seed": 1,
        "accuracy": 0.6708554857419043,
        "invalid": 45,
        "cutoff": 0,
        "wrong_answers_caught": 116
       },
       {
        "run": "grading:chat-qwen3.5-2b-nothink-seed2-v1",
        "seed": 2,
        "accuracy": 0.6718221362977284,
        "invalid": 43,
        "cutoff": 0,
        "wrong_answers_caught": 116
       }
      ],
      "invalid_rate_mean": 0.020138553246334782
     },
     {
      "name": "OneJev1.6-0.8B",
      "display_name": "OneJev1.6-0.8B",
      "tag": "",
      "model_key": "onejev1.6-0.8b",
      "maker": "OmniJev",
      "kind": "decision",
      "repo": "OmniJev/OneJev1.6-0.8B",
      "commit": "bcd88a382f314b26cb7b1b7f369c1ed3502ee7a0",
      "params_total": 0.8,
      "params_active": 0.8,
      "quantization": "BF16",
      "score_0_100": 50.0483325277912,
      "raw": {
       "metric": "accuracy",
       "value": 0.6669888835186081,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.040721618656355756,
      "improvement_points": null,
      "runs": [
       "grading:site-onejev1.6-0.8b-v1"
      ],
      "notes": "OmniJev/OneJev1.6-0.8B@bcd88a38, the server its Logic & Math runs use, BF16, on one DGX Spark (ours), self-hosted; server probabilities are rounded to six decimal places and retained without renormalization; ECE uses the reported maximum probability; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.024637469203680373,
      "gpu_cost_usd": null
     },
     {
      "name": "d1-2B",
      "display_name": "d1-2B",
      "tag": "",
      "model_key": "liquid-d1-2b",
      "maker": "Liquid AI",
      "kind": "decision",
      "repo": "LiquidAI/d1-2B",
      "commit": "10e7671d44e4b37700020b1eed3f594c5f2af0b4",
      "params_total": 1.92,
      "params_active": 1.92,
      "quantization": "BF16",
      "score_0_100": 49.613339777670376,
      "raw": {
       "metric": "accuracy",
       "value": 0.6640889318511358,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.07187269491787361,
      "improvement_points": null,
      "runs": [
       "grading:site-liquid-d1-2b-v1"
      ],
      "notes": "LiquidAI/d1-2B@10e7671d, d1_server (Liquid's model code) on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.014600491343951257,
      "gpu_cost_usd": null
     },
     {
      "name": "StartLux-Decision-v1.1-0.8B-Instant",
      "display_name": "StartLux-Decision-v1.1-0.8B-Instant",
      "tag": "",
      "model_key": "startlux-decision-v1.1-0.8b-instant",
      "maker": "StartLux",
      "kind": "decision",
      "repo": "startlux-models/StartLux-Decision-v1.1-0.8B-Instant",
      "commit": "6496ae641cc71b149ce9dad69bd97f2a44e7bf11",
      "params_total": 0.8,
      "params_active": 0.8,
      "quantization": "BF16",
      "score_0_100": 48.88835186080232,
      "raw": {
       "metric": "accuracy",
       "value": 0.6592556790720154,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.056035763218873734,
      "improvement_points": null,
      "runs": [
       "grading:site-startlux-decision-v1.1-0.8b-instant-v1"
      ],
      "notes": "startlux-models/StartLux-Decision-v1.1-0.8B-Instant@6496ae64, the server its Logic & Math runs use, BF16, on one DGX Spark (ours), self-hosted; licence CC-BY-NC-4.0 (research use); one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.030659771969914984,
      "gpu_cost_usd": null
     },
     {
      "name": "GLiDE 0.4B",
      "display_name": "GLiDE 0.4B",
      "tag": "",
      "model_key": "glide-0.4b",
      "maker": "Fastino",
      "kind": "decision",
      "repo": "fastino/GLiDE-0.4B",
      "commit": "6bda7ab91aeb36dcad6d27828a8871e548c7fc71",
      "params_total": 0.4,
      "params_active": 0.4,
      "quantization": "BF16",
      "score_0_100": 44.393426776220394,
      "raw": {
       "metric": "accuracy",
       "value": 0.6292895118414693,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.07677301660636211,
      "improvement_points": null,
      "runs": [
       "grading:site-glide-0.4b-v1"
      ],
      "notes": "fastino/GLiDE-0.4B@6bda7ab9, the server its Logic & Math runs use, BF16, on one DGX Spark (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.023397810828577028,
      "gpu_cost_usd": null
     },
     {
      "name": "GLiClass v4 Edge",
      "display_name": "GLiClass v4 Edge",
      "tag": "",
      "model_key": "gliclass-v4-edge",
      "maker": "Knowledgator",
      "kind": "decision",
      "repo": "knowledgator/gliclass-v4-edge",
      "commit": "885af1af8a07edf4ed34df83ca0d995ed8201283",
      "params_total": 0.4,
      "params_active": 0.4,
      "quantization": "BF16",
      "score_0_100": 44.32092798453359,
      "raw": {
       "metric": "accuracy",
       "value": 0.6288061865635572,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.09842112173409612,
      "improvement_points": null,
      "runs": [
       "grading:site-gliclass-v4-edge-v1"
      ],
      "notes": "knowledgator/gliclass-v4-edge@885af1af, the server its Logic & Math runs use, BF16, on one DGX Spark (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.07387427885198607,
      "gpu_cost_usd": null
     },
     {
      "name": "Decider 1",
      "display_name": "Decider 1",
      "tag": "",
      "model_key": "meragpt-decider-1",
      "maker": "meraGPT",
      "kind": "decision",
      "repo": null,
      "commit": null,
      "params_total": null,
      "params_active": null,
      "quantization": null,
      "score_0_100": 43.30594490091833,
      "raw": {
       "metric": "accuracy",
       "value": 0.6220396326727888,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.059314403093281794,
      "improvement_points": null,
      "runs": [
       "grading:site-meragpt-decider-1-v1"
      ],
      "notes": "meraGPT decisions API (meragpt.com, model decider-1); one question per request, 8 requests at a time (latency not comparable across different concurrency)",
      "gpu_hours": null,
      "gpu_cost_usd": null
     },
     {
      "name": "Raon-1 nano",
      "display_name": "Raon-1 nano",
      "tag": "thinking off · chat",
      "model_key": "raon-1-nano-nothink",
      "maker": "KRAFTON",
      "kind": "chat",
      "repo": "KRAFTON/Raon-1-nano",
      "commit": "fcf2d1d55e84e2f9fbe7b5d724dcfc4d861cfcee",
      "params_total": 3.1,
      "params_active": 1.5,
      "quantization": "BF16",
      "score_0_100": 40.647655872402126,
      "raw": {
       "metric": "accuracy",
       "value": 0.6043177058160142,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": null,
      "improvement_points": null,
      "runs": [
       "grading:chat-raon-1-nano-nothink-v1",
       "grading:chat-raon-1-nano-nothink-seed1-v1",
       "grading:chat-raon-1-nano-nothink-seed2-v1"
      ],
      "notes": "KRAFTON/Raon-1-nano@fcf2d1d5, BF16, KRAFTON vLLM fork krafton-ai/vllm@13a5dd87 (not a vLLM release) on one RTX PRO 6000 (ours), self-hosted; one question per request, 30 requests at a time from the client (latency not comparable across different concurrency); mean of 3 full runs, seeds 0, 1, 2 (min-max across seeds, not a confidence interval); answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; replies without an answer marker count wrong: mean rate 0.03% (per seed in seed_runs; cut off at the token limit per seed: 0, 1, 0); seed 0 is the published single run (every reply kept); seeds 1 and 2 are new runs (16 Nov, every reply kept) with the same model, sampling settings, requests at a time and server launcher, only the seed changed, their server settings recorded; gpu_hours is the mean of the three runs",
      "gpu_hours": 0.11989139371652192,
      "gpu_cost_usd": null,
      "accuracy_basis": "mean of 3 seeds",
      "seed_min": 0.6036732721121315,
      "seed_max": 0.6051232479458676,
      "seed_runs": [
       {
        "run": "grading:chat-raon-1-nano-nothink-v1",
        "seed": 0,
        "accuracy": 0.6051232479458676,
        "invalid": 0,
        "cutoff": 0,
        "wrong_answers_caught": 65
       },
       {
        "run": "grading:chat-raon-1-nano-nothink-seed1-v1",
        "seed": 1,
        "accuracy": 0.6041565973900435,
        "invalid": 1,
        "cutoff": 1,
        "wrong_answers_caught": 68
       },
       {
        "run": "grading:chat-raon-1-nano-nothink-seed2-v1",
        "seed": 2,
        "accuracy": 0.6036732721121315,
        "invalid": 1,
        "cutoff": 0,
        "wrong_answers_caught": 63
       }
      ],
      "invalid_rate_mean": 0.0003222168519413565
     },
     {
      "name": "KnowLine-0.8B-Gen1",
      "display_name": "KnowLine-0.8B-Gen1",
      "tag": "",
      "model_key": "knowline-0.8b-gen1",
      "maker": "PelaAI",
      "kind": "decision",
      "repo": "PelaAI/KnowLine-0.8B-Gen1",
      "commit": "6b27831f7b783c03278bd946d4d8f4f7a98a3798",
      "params_total": 0.8,
      "params_active": 0.8,
      "quantization": "BF16",
      "score_0_100": 38.52102464958917,
      "raw": {
       "metric": "accuracy",
       "value": 0.5901401643305945,
       "questions": 2069,
       "options": 3
      },
      "calibration_error": 0.14642663407717668,
      "improvement_points": null,
      "runs": [
       "grading:site-knowline-0.8b-gen1-v1"
      ],
      "notes": "PelaAI/KnowLine-0.8B-Gen1@6b27831f, the server its Logic & Math runs use, BF16, on one DGX Spark (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
      "gpu_hours": 0.04669099587283346,
      "gpu_cost_usd": null
     }
    ]
   },
   "sha256": "f2ebb5ed58d87c33daff7cf166553c5f6a70a4b176d6ba4802078b1d75ba82a5"
  },
  "grading-metrics": {
   "url": "https://zekiworks.github.io/analysis/grading/metrics.json",
   "published": true,
   "seen": "2026-11-16",
   "file": {
    "bench": "grading",
    "updated": "2026-11-16T19:44:24Z",
    "feed": {
     "file": "bench-scores.json",
     "sha256": "f2ebb5ed58d87c33daff7cf166553c5f6a70a4b176d6ba4802078b1d75ba82a5"
    },
    "cohort_pairs": 2069,
    "label_pairs": {
     "reference": 942,
     "other": 329,
     "none": 798
    },
    "false_alarm_pairs": 1740,
    "definitions": {
     "latency": "client-side wall time per request, request sent to answer parsed; compare only rows with the same requests_in_flight",
     "api_cost_per_1000_usd_list_price": "billed input tokens x the provider's list price (price_source, price_date), per 1,000 questions; for a chat model billed per token through OpenRouter, input and output (reasoning included) tokens at its list prices; not an invoice; null on own GPU and on a subscription route, and null with its rate, source and date where no public price page backs that exact model's rate (checked 13 Nov): unverified, not zero",
     "gpu_seconds_per_1000": "measured wall time of the whole run on one GPU (model load excluded) over all pairs it answered, per 1,000 questions: a rate, not a cohort share; own GPU only, not comparable with API dollars",
     "recall": "of all gold pairs with that label, the share labelled so; an invalid reply counts as missed",
     "wrong_answer_recall": "recall_other: texts stating a conflicting answer that were flagged 'other'",
     "false_alarm_rate": "gold 'reference' or 'none' pairs labelled 'other', over false_alarm_pairs; invalid replies are not counted as correct rejections, so 1 - false_alarm_rate is not an accuracy; read it beside invalid",
     "invalid": "replies without a usable label; for chat, no last matching ANSWER marker in the visible output under answer rule v2 (keyword in any letter case, option an uppercase A, B or C); counted wrong",
     "measurement_gap": "present only where a run was interrupted and resumed: latency and gpu_seconds_per_1000 are then null, with the reason; scores are unaffected",
     "probabilities": "this run's records hold per-option probabilities for every pair; says nothing about other runs of the model; no confidence is derived for chat models; calibration_error stays in bench-scores.json",
     "seed_groups": "a row with 'runs' covers three full runs with seeds 0, 1, 2 under its first run's id: recalls and false_alarm_rate are the mean of the three, latency is over all their requests, gpu_seconds_per_1000 the mean rate; 'invalid' is null and each seed's count is in invalid_per_seed"
    },
    "models": [
     {
      "run": "grading:d1-ultra-v1",
      "model_key": "liquid-d1-ultra",
      "latency_s_median": 0.485,
      "latency_s_p90": 0.903,
      "requests_in_flight": 8,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": 0.0351,
      "price_usd_per_million_input": 0.042,
      "price_source": "https://www.liquid.ai/blog/d1-ultra",
      "price_date": "2026-10-16",
      "gpu_seconds_per_1000": null,
      "recall_reference": 0.9565,
      "recall_other": 0.8085,
      "recall_none": 0.8659,
      "wrong_answer_recall": 0.8085,
      "false_alarm_rate": 0.0454,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:gemini-v1",
      "model_key": "gemini-decision-1",
      "latency_s_median": 0.203,
      "latency_s_p90": 0.291,
      "requests_in_flight": 8,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": 0.0935,
      "price_usd_per_million_input": 0.1,
      "price_source": "https://ai.google.dev/gemini-api/docs/pricing",
      "price_date": "2026-11-13",
      "gpu_seconds_per_1000": null,
      "recall_reference": 0.965,
      "recall_other": 0.5106,
      "recall_none": 0.8459,
      "wrong_answer_recall": 0.5106,
      "false_alarm_rate": 0.0178,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-liquid-d1-v1",
      "model_key": "liquid-d1",
      "latency_s_median": 0.265,
      "latency_s_p90": 0.601,
      "requests_in_flight": 8,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": null,
      "recall_reference": 0.9108,
      "recall_other": 0.6626,
      "recall_none": 0.8847,
      "wrong_answer_recall": 0.6626,
      "false_alarm_rate": 0.0529,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-liquid-d1.1-v1",
      "model_key": "liquid-d1.1",
      "latency_s_median": 0.25,
      "latency_s_p90": 0.453,
      "requests_in_flight": 8,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": 0.0167,
      "price_usd_per_million_input": 0.02,
      "price_source": "https://www.liquid.ai/pricing",
      "price_date": "2026-10-24",
      "gpu_seconds_per_1000": null,
      "recall_reference": 0.9246,
      "recall_other": 0.7052,
      "recall_none": 0.9085,
      "wrong_answer_recall": 0.7052,
      "false_alarm_rate": 0.0414,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-liquid-d1.2-ultra-v1",
      "model_key": "liquid-d1.2-ultra",
      "latency_s_median": 0.229,
      "latency_s_p90": 0.347,
      "requests_in_flight": 8,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": 0.0351,
      "price_usd_per_million_input": 0.042,
      "price_source": "https://www.liquid.ai/blog/d1-2",
      "price_date": "2026-11-16",
      "gpu_seconds_per_1000": null,
      "recall_reference": 0.9193,
      "recall_other": 0.8541,
      "recall_none": 0.9211,
      "wrong_answer_recall": 0.8541,
      "false_alarm_rate": 0.0443,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-fastino-glide-v1",
      "model_key": "fastino-glide",
      "latency_s_median": 0.299,
      "latency_s_p90": 0.558,
      "requests_in_flight": 8,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": null,
      "recall_reference": 0.9639,
      "recall_other": 0.6869,
      "recall_none": 0.9135,
      "wrong_answer_recall": 0.6869,
      "false_alarm_rate": 0.0172,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-subq-snap-1.1-v1",
      "model_key": "subq-snap-1.1",
      "latency_s_median": 0.394,
      "latency_s_p90": 0.412,
      "requests_in_flight": 8,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": null,
      "recall_reference": 0.9193,
      "recall_other": 0.7812,
      "recall_none": 0.9248,
      "wrong_answer_recall": 0.7812,
      "false_alarm_rate": 0.0414,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-jev-1.13.0-v1",
      "model_key": "jev-1.13.0",
      "latency_s_median": 0.169,
      "latency_s_p90": 0.19,
      "requests_in_flight": 8,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": 0.0487,
      "price_usd_per_million_input": 0.042,
      "price_source": "https://typesafe.ai/blog/introducing-system-one-models-and-jev",
      "price_date": "2026-10-08",
      "gpu_seconds_per_1000": null,
      "recall_reference": 0.9427,
      "recall_other": 0.7994,
      "recall_none": 0.8559,
      "wrong_answer_recall": 0.7994,
      "false_alarm_rate": 0.0598,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-jev-1.14.0-v1",
      "model_key": "jev-1.14.0",
      "latency_s_median": 0.176,
      "latency_s_p90": 0.203,
      "requests_in_flight": 8,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": 0.0487,
      "price_usd_per_million_input": 0.042,
      "price_source": "https://docs.typesafe.ai/models",
      "price_date": "2026-10-16",
      "gpu_seconds_per_1000": null,
      "recall_reference": 0.9406,
      "recall_other": 0.7204,
      "recall_none": 0.9023,
      "wrong_answer_recall": 0.7204,
      "false_alarm_rate": 0.0351,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-microsoft-decision-1-v1",
      "model_key": "microsoft-decision-1",
      "latency_s_median": 0.324,
      "latency_s_p90": 0.377,
      "requests_in_flight": 8,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": 0.0363,
      "price_usd_per_million_input": 0.042,
      "price_source": "https://openrouter.ai/microsoft/microsoft-decision-1",
      "price_date": "2026-10-17",
      "gpu_seconds_per_1000": null,
      "recall_reference": 0.8949,
      "recall_other": 0.8237,
      "recall_none": 0.9185,
      "wrong_answer_recall": 0.8237,
      "false_alarm_rate": 0.0649,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-meragpt-decider-1-v1",
      "model_key": "meragpt-decider-1",
      "latency_s_median": 0.652,
      "latency_s_p90": 0.754,
      "requests_in_flight": 8,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": 0.0353,
      "price_usd_per_million_input": 0.04,
      "price_source": "https://meragpt.com/docs/models",
      "price_date": "2026-10-19",
      "gpu_seconds_per_1000": null,
      "recall_reference": 0.7017,
      "recall_other": 0.3191,
      "recall_none": 0.6529,
      "wrong_answer_recall": 0.3191,
      "false_alarm_rate": 0.1759,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-liquid-d1-2b-v1",
      "model_key": "liquid-d1-2b",
      "latency_s_median": 0.019,
      "latency_s_p90": 0.025,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 25.4,
      "recall_reference": 0.8429,
      "recall_other": 0.8571,
      "recall_none": 0.3734,
      "wrong_answer_recall": 0.8571,
      "false_alarm_rate": 0.3218,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-glide-2b-v1",
      "model_key": "glide-2b",
      "latency_s_median": 0.035,
      "latency_s_p90": 0.041,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 43.5,
      "recall_reference": 0.897,
      "recall_other": 0.6079,
      "recall_none": 0.6892,
      "wrong_answer_recall": 0.6079,
      "false_alarm_rate": 0.108,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-knowline-31b-gen1.5-v1",
      "model_key": "knowline-31b-gen1.5",
      "latency_s_median": 2.545,
      "latency_s_p90": 3.881,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 2613.4,
      "recall_reference": 0.9724,
      "recall_other": 0.8359,
      "recall_none": 0.9073,
      "wrong_answer_recall": 0.8359,
      "false_alarm_rate": 0.0305,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-glide-27b-v1",
      "model_key": "glide-27b",
      "latency_s_median": 1.035,
      "latency_s_p90": 1.951,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 1204.8,
      "recall_reference": 0.9713,
      "recall_other": 0.7477,
      "recall_none": 0.8972,
      "wrong_answer_recall": 0.7477,
      "false_alarm_rate": 0.0149,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-sieve-31b-v3-v1",
      "model_key": "sieve-31b-v3",
      "latency_s_median": 2.673,
      "latency_s_p90": 3.931,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 2823.7,
      "recall_reference": 0.9862,
      "recall_other": 0.7933,
      "recall_none": 0.896,
      "wrong_answer_recall": 0.7933,
      "false_alarm_rate": 0.0213,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-fastino-glide-v1.2-v1",
      "model_key": "fastino-glide-v1.2",
      "latency_s_median": 0.234,
      "latency_s_p90": 0.357,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 253.6,
      "recall_reference": 0.966,
      "recall_other": 0.693,
      "recall_none": 0.906,
      "wrong_answer_recall": 0.693,
      "false_alarm_rate": 0.0184,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-reax-1.3-v1",
      "model_key": "reax-1.3",
      "latency_s_median": 0.677,
      "latency_s_p90": 1.047,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 745.3,
      "recall_reference": 0.9618,
      "recall_other": 0.6748,
      "recall_none": 0.9536,
      "wrong_answer_recall": 0.6748,
      "false_alarm_rate": 0.0121,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-turion-jev-31b-v1",
      "model_key": "turion-jev-31b",
      "latency_s_median": 1.123,
      "latency_s_p90": 1.956,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 1328.9,
      "recall_reference": 0.9682,
      "recall_other": 0.6626,
      "recall_none": 0.9424,
      "wrong_answer_recall": 0.6626,
      "false_alarm_rate": 0.0167,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-perplexity-decider-27b-v1.3-v1",
      "model_key": "perplexity-decider-27b-v1.3",
      "latency_s_median": 0.223,
      "latency_s_p90": 0.34,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 245.0,
      "recall_reference": 0.9512,
      "recall_other": 0.8267,
      "recall_none": 0.891,
      "wrong_answer_recall": 0.8267,
      "false_alarm_rate": 0.0351,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-perplexity-decider-flash-v1.3-v1",
      "model_key": "perplexity-decider-flash-v1.3",
      "latency_s_median": 0.055,
      "latency_s_p90": 0.081,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 61.2,
      "recall_reference": 0.9374,
      "recall_other": 0.6748,
      "recall_none": 0.8421,
      "wrong_answer_recall": 0.6748,
      "false_alarm_rate": 0.0557,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-perplexity-decider-flash-v1.2-v1",
      "model_key": "perplexity-decider-flash-v1.2",
      "latency_s_median": 0.058,
      "latency_s_p90": 0.085,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 64.6,
      "recall_reference": 0.9406,
      "recall_other": 0.4772,
      "recall_none": 0.891,
      "wrong_answer_recall": 0.4772,
      "false_alarm_rate": 0.0397,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-glide-9b-v1",
      "model_key": "glide-9b",
      "latency_s_median": 0.08,
      "latency_s_p90": 0.113,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 90.4,
      "recall_reference": 0.9459,
      "recall_other": 0.6991,
      "recall_none": 0.881,
      "wrong_answer_recall": 0.6991,
      "false_alarm_rate": 0.0379,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-elephant-1-v1",
      "model_key": "elephant-1",
      "latency_s_median": 0.09,
      "latency_s_p90": 0.111,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 98.0,
      "recall_reference": 0.9374,
      "recall_other": 0.4103,
      "recall_none": 0.9173,
      "wrong_answer_recall": 0.4103,
      "false_alarm_rate": 0.019,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-karpo-decision-31b-v1",
      "model_key": "karpo-decision-31b",
      "latency_s_median": 0.216,
      "latency_s_p90": 0.311,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 230.5,
      "recall_reference": 0.9798,
      "recall_other": 0.8085,
      "recall_none": 0.9298,
      "wrong_answer_recall": 0.8085,
      "false_alarm_rate": 0.0167,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-copernicus-31b-v1",
      "model_key": "copernicus-31b",
      "latency_s_median": 0.516,
      "latency_s_p90": 0.829,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 577.0,
      "recall_reference": 0.9883,
      "recall_other": 0.8146,
      "recall_none": 0.8997,
      "wrong_answer_recall": 0.8146,
      "false_alarm_rate": 0.0253,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-knowline-9b-gen1-v1",
      "model_key": "knowline-9b-gen1",
      "latency_s_median": 0.097,
      "latency_s_p90": 0.143,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 104.7,
      "recall_reference": 0.8439,
      "recall_other": 0.5076,
      "recall_none": 0.9749,
      "wrong_answer_recall": 0.5076,
      "false_alarm_rate": 0.0379,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-adjudex-9b-v5-v1",
      "model_key": "adjudex-9b-v5",
      "latency_s_median": 0.065,
      "latency_s_p90": 0.092,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 71.4,
      "recall_reference": 0.9225,
      "recall_other": 0.6505,
      "recall_none": 0.8972,
      "wrong_answer_recall": 0.6505,
      "false_alarm_rate": 0.0489,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-knowline-4b-gen5.5-v1",
      "model_key": "knowline-4b-gen5.5",
      "latency_s_median": 0.045,
      "latency_s_p90": 0.064,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 49.0,
      "recall_reference": 0.9055,
      "recall_other": 0.6201,
      "recall_none": 0.9336,
      "wrong_answer_recall": 0.6201,
      "false_alarm_rate": 0.0339,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-knowline-9b-gen2-v1",
      "model_key": "knowline-9b-gen2",
      "latency_s_median": 0.101,
      "latency_s_p90": 0.158,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 128.5,
      "recall_reference": 0.8928,
      "recall_other": 0.3587,
      "recall_none": 0.9637,
      "wrong_answer_recall": 0.3587,
      "false_alarm_rate": 0.0144,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-knowline-12b-gen1-v1",
      "model_key": "knowline-12b-gen1",
      "latency_s_median": 0.163,
      "latency_s_p90": 0.247,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 194.6,
      "recall_reference": 0.9331,
      "recall_other": 0.6109,
      "recall_none": 0.9085,
      "wrong_answer_recall": 0.6109,
      "false_alarm_rate": 0.0356,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-knowline-31b-gen2-v1",
      "model_key": "knowline-31b-gen2",
      "latency_s_median": 0.389,
      "latency_s_p90": 0.612,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 449.4,
      "recall_reference": 0.9682,
      "recall_other": 0.8632,
      "recall_none": 0.881,
      "wrong_answer_recall": 0.8632,
      "false_alarm_rate": 0.0362,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-jeffan-9b-1211-v1",
      "model_key": "jeffan-9b-1211",
      "latency_s_median": 0.093,
      "latency_s_p90": 0.138,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 99.6,
      "recall_reference": 0.9172,
      "recall_other": 0.5988,
      "recall_none": 0.9687,
      "wrong_answer_recall": 0.5988,
      "false_alarm_rate": 0.0247,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-cerno-9b-v1.2-v1",
      "model_key": "cerno-9b-v1.2",
      "latency_s_median": 0.114,
      "latency_s_p90": 0.177,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 126.3,
      "recall_reference": 0.9119,
      "recall_other": 0.6535,
      "recall_none": 0.9336,
      "wrong_answer_recall": 0.6535,
      "false_alarm_rate": 0.0305,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-chinese-decision-pointer-4b-v1",
      "model_key": "chinese-decision-pointer-4b",
      "latency_s_median": 0.069,
      "latency_s_p90": 0.107,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 76.8,
      "recall_reference": 0.9066,
      "recall_other": 0.69,
      "recall_none": 0.8596,
      "wrong_answer_recall": 0.69,
      "false_alarm_rate": 0.0603,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-logits-sigma-1-duo-v1",
      "model_key": "logits-sigma-1-duo",
      "latency_s_median": 0.641,
      "latency_s_p90": 1.014,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 723.3,
      "recall_reference": 0.9437,
      "recall_other": 0.6383,
      "recall_none": 0.9586,
      "wrong_answer_recall": 0.6383,
      "false_alarm_rate": 0.0086,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-startlux-decision-v1.1-4b-v1",
      "model_key": "startlux-decision-v1.1-4b",
      "latency_s_median": 0.121,
      "latency_s_p90": 0.186,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 140.3,
      "recall_reference": 0.9023,
      "recall_other": 0.8936,
      "recall_none": 0.8421,
      "wrong_answer_recall": 0.8936,
      "false_alarm_rate": 0.0828,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-startlux-decision-v1.1-9b-v1",
      "model_key": "startlux-decision-v1.1-9b",
      "latency_s_median": 0.229,
      "latency_s_p90": 0.357,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 267.5,
      "recall_reference": 0.9034,
      "recall_other": 0.8875,
      "recall_none": 0.7945,
      "wrong_answer_recall": 0.8875,
      "false_alarm_rate": 0.0954,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-verdin-4b-v1.1-v1",
      "model_key": "verdin-4b-v1.1",
      "latency_s_median": 0.049,
      "latency_s_p90": 0.065,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 54.1,
      "recall_reference": 0.9501,
      "recall_other": 0.4681,
      "recall_none": 0.8521,
      "wrong_answer_recall": 0.4681,
      "false_alarm_rate": 0.0374,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-valen-decision-1.5-9b-v1",
      "model_key": "valen-decision-1.5-9b",
      "latency_s_median": 0.071,
      "latency_s_p90": 0.095,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 78.4,
      "recall_reference": 0.9342,
      "recall_other": 0.3982,
      "recall_none": 0.881,
      "wrong_answer_recall": 0.3982,
      "false_alarm_rate": 0.0276,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-vera-medium-omni-v1",
      "model_key": "vera-medium-omni",
      "latency_s_median": 0.057,
      "latency_s_p90": 0.063,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 59.6,
      "recall_reference": 0.9278,
      "recall_other": 0.7599,
      "recall_none": 0.9023,
      "wrong_answer_recall": 0.7599,
      "false_alarm_rate": 0.0569,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-dexy-decision-9b-v2-v1",
      "model_key": "dexy-decision-9b-v2",
      "latency_s_median": 0.063,
      "latency_s_p90": 0.088,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 71.5,
      "recall_reference": 0.9268,
      "recall_other": 0.6474,
      "recall_none": 0.9048,
      "wrong_answer_recall": 0.6474,
      "false_alarm_rate": 0.0391,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-clef-flash-v1",
      "model_key": "clef-flash",
      "latency_s_median": 0.152,
      "latency_s_p90": 0.18,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 155.7,
      "recall_reference": 0.9257,
      "recall_other": 0.3769,
      "recall_none": 0.8221,
      "wrong_answer_recall": 0.3769,
      "false_alarm_rate": 0.0391,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-cygnet-v1",
      "model_key": "cygnet",
      "latency_s_median": 0.087,
      "latency_s_p90": 0.128,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 92.9,
      "recall_reference": 0.9437,
      "recall_other": 0.2918,
      "recall_none": 0.8897,
      "wrong_answer_recall": 0.2918,
      "false_alarm_rate": 0.0207,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-winnow-12b-v1",
      "model_key": "winnow-12b",
      "latency_s_median": 0.122,
      "latency_s_p90": 0.178,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 132.1,
      "recall_reference": 0.9427,
      "recall_other": 0.5167,
      "recall_none": 0.8622,
      "wrong_answer_recall": 0.5167,
      "false_alarm_rate": 0.0379,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-metask-jev-4b-v1",
      "model_key": "metask-jev-4b",
      "latency_s_median": 0.116,
      "latency_s_p90": 0.186,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 133.9,
      "recall_reference": 0.9618,
      "recall_other": 0.231,
      "recall_none": 0.7005,
      "wrong_answer_recall": 0.231,
      "false_alarm_rate": 0.0339,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-strands-decider-2b-v1",
      "model_key": "strands-decider-2b",
      "latency_s_median": 0.038,
      "latency_s_p90": 0.054,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 43.1,
      "recall_reference": 0.9639,
      "recall_other": 0.0456,
      "recall_none": 0.7506,
      "wrong_answer_recall": 0.0456,
      "false_alarm_rate": 0.008,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-glide-0.8b-v1",
      "model_key": "glide-0.8b",
      "latency_s_median": 0.048,
      "latency_s_p90": 0.075,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 63.3,
      "recall_reference": 0.8992,
      "recall_other": 0.3131,
      "recall_none": 0.7243,
      "wrong_answer_recall": 0.3131,
      "false_alarm_rate": 0.0655,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-glide-0.4b-v1",
      "model_key": "glide-0.4b",
      "latency_s_median": 0.028,
      "latency_s_p90": 0.044,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 40.7,
      "recall_reference": 0.8397,
      "recall_other": 0.31,
      "recall_none": 0.5125,
      "wrong_answer_recall": 0.31,
      "false_alarm_rate": 0.1684,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-startlux-decision-v1.1-2b-v1",
      "model_key": "startlux-decision-v1.1-2b",
      "latency_s_median": 0.063,
      "latency_s_p90": 0.087,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 78.2,
      "recall_reference": 0.8875,
      "recall_other": 0.6748,
      "recall_none": 0.6792,
      "wrong_answer_recall": 0.6748,
      "false_alarm_rate": 0.1397,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-startlux-decision-v1.1-0.8b-instant-v1",
      "model_key": "startlux-decision-v1.1-0.8b-instant",
      "latency_s_median": 0.042,
      "latency_s_p90": 0.059,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 53.3,
      "recall_reference": 0.8386,
      "recall_other": 0.7234,
      "recall_none": 0.4211,
      "wrong_answer_recall": 0.7234,
      "false_alarm_rate": 0.2649,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-onejev1.6-0.8b-v1",
      "model_key": "onejev1.6-0.8b",
      "latency_s_median": 0.031,
      "latency_s_p90": 0.043,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 42.9,
      "recall_reference": 0.8227,
      "recall_other": 0.3343,
      "recall_none": 0.6203,
      "wrong_answer_recall": 0.3343,
      "false_alarm_rate": 0.1437,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-knowline-2b-gen1-v1",
      "model_key": "knowline-2b-gen1",
      "latency_s_median": 0.109,
      "latency_s_p90": 0.17,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 129.8,
      "recall_reference": 0.9257,
      "recall_other": 0.1641,
      "recall_none": 0.7594,
      "wrong_answer_recall": 0.1641,
      "false_alarm_rate": 0.027,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-knowline-0.8b-gen1-v1",
      "model_key": "knowline-0.8b-gen1",
      "latency_s_median": 0.068,
      "latency_s_p90": 0.103,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 81.2,
      "recall_reference": 0.9798,
      "recall_other": 0.1003,
      "recall_none": 0.3321,
      "wrong_answer_recall": 0.1003,
      "false_alarm_rate": 0.0356,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:site-gliclass-v4-edge-v1",
      "model_key": "gliclass-v4-edge",
      "latency_s_median": 0.075,
      "latency_s_p90": 0.18,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 128.5,
      "recall_reference": 0.9246,
      "recall_other": 0.3495,
      "recall_none": 0.3947,
      "wrong_answer_recall": 0.3495,
      "false_alarm_rate": 0.1057,
      "invalid": 0,
      "probabilities": true
     },
     {
      "run": "grading:chat-qwen3.5-4b-nothink-v1",
      "model_key": "qwen3.5-4b-nothink",
      "runs": [
       "grading:chat-qwen3.5-4b-nothink-v1",
       "grading:chat-qwen3.5-4b-nothink-seed1-v1",
       "grading:chat-qwen3.5-4b-nothink-seed2-v1"
      ],
      "aggregation": "mean of 3 seeds; latency over all requests of the three runs",
      "latency_s_median": 3.487,
      "latency_s_p90": 20.434,
      "requests_in_flight": 64,
      "requests_measured": 6207,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 148.4,
      "recall_reference": 0.9441,
      "recall_other": 0.612,
      "recall_none": 0.6145,
      "wrong_answer_recall": 0.612,
      "false_alarm_rate": 0.0764,
      "invalid": null,
      "invalid_per_seed": [
       20,
       25,
       15
      ],
      "probabilities": false
     },
     {
      "run": "grading:chat-qwen3.5-2b-nothink-v1",
      "model_key": "qwen3.5-2b-nothink",
      "runs": [
       "grading:chat-qwen3.5-2b-nothink-v1",
       "grading:chat-qwen3.5-2b-nothink-seed1-v1",
       "grading:chat-qwen3.5-2b-nothink-seed2-v1"
      ],
      "aggregation": "mean of 3 seeds; latency over all requests of the three runs",
      "latency_s_median": 1.53,
      "latency_s_p90": 13.725,
      "requests_in_flight": 64,
      "requests_measured": 6207,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 91.6,
      "recall_reference": 0.804,
      "recall_other": 0.3445,
      "recall_none": 0.6416,
      "wrong_answer_recall": 0.3445,
      "false_alarm_rate": 0.0776,
      "invalid": null,
      "invalid_per_seed": [
       37,
       45,
       43
      ],
      "probabilities": false
     },
     {
      "run": "grading:chat-gemma-4-e4b-think-v1",
      "model_key": "gemma-4-e4b-think",
      "latency_s_median": 9.38,
      "latency_s_p90": 11.907,
      "requests_in_flight": 30,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 325.5,
      "recall_reference": 0.9119,
      "recall_other": 0.6748,
      "recall_none": 0.8434,
      "wrong_answer_recall": 0.6748,
      "false_alarm_rate": 0.0448,
      "invalid": 2,
      "probabilities": false
     },
     {
      "run": "grading:chat-raon-1-nano-nothink-v1",
      "model_key": "raon-1-nano-nothink",
      "runs": [
       "grading:chat-raon-1-nano-nothink-v1",
       "grading:chat-raon-1-nano-nothink-seed1-v1",
       "grading:chat-raon-1-nano-nothink-seed2-v1"
      ],
      "aggregation": "mean of 3 seeds; latency over all requests of the three runs",
      "latency_s_median": 3.029,
      "latency_s_p90": 7.765,
      "requests_in_flight": 30,
      "requests_measured": 6207,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 208.6,
      "recall_reference": 0.649,
      "recall_other": 0.1986,
      "recall_none": 0.7189,
      "wrong_answer_recall": 0.1986,
      "false_alarm_rate": 0.0203,
      "invalid": null,
      "invalid_per_seed": [
       0,
       1,
       1
      ],
      "probabilities": false
     },
     {
      "run": "grading:chat-granite-4.2-3b-think-v1",
      "model_key": "granite-4.2-3b-think",
      "latency_s_median": 19.592,
      "latency_s_p90": 64.465,
      "requests_in_flight": 30,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 1050.3,
      "recall_reference": 0.8832,
      "recall_other": 0.7903,
      "recall_none": 0.807,
      "wrong_answer_recall": 0.7903,
      "false_alarm_rate": 0.0902,
      "invalid": 0,
      "probabilities": false
     },
     {
      "run": "grading:chat-lfm2.5-8b-a1b-think-v1",
      "model_key": "lfm2.5-8b-a1b-think",
      "latency_s_median": 9.579,
      "latency_s_p90": 15.194,
      "requests_in_flight": 30,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 351.9,
      "recall_reference": 0.7951,
      "recall_other": 0.0881,
      "recall_none": 0.8872,
      "wrong_answer_recall": 0.0881,
      "false_alarm_rate": 0.008,
      "invalid": 0,
      "probabilities": false
     },
     {
      "run": "grading:chat-gemini-4-hydrogen-v1",
      "model_key": "gemini-4-hydrogen",
      "latency_s_median": 6.372,
      "latency_s_p90": 19.169,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": null,
      "recall_reference": 0.8843,
      "recall_other": 0.6869,
      "recall_none": 0.9837,
      "wrong_answer_recall": 0.6869,
      "false_alarm_rate": 0.0218,
      "invalid": 0,
      "probabilities": false
     },
     {
      "run": "grading:chat-inkling-2-small-v1",
      "model_key": "inkling-2-small",
      "latency_s_median": 2.702,
      "latency_s_p90": 16.351,
      "requests_in_flight": 1,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": 1.4933,
      "price_usd_per_million_input": 0.3,
      "price_source": "https://openrouter.ai/api/v1/models",
      "price_date": "2026-11-16",
      "gpu_seconds_per_1000": null,
      "recall_reference": 0.9724,
      "recall_other": 0.7356,
      "recall_none": 0.9035,
      "wrong_answer_recall": 0.7356,
      "false_alarm_rate": 0.0241,
      "invalid": 0,
      "probabilities": false
     },
     {
      "run": "grading:chat-lfm3-1.7b-seed0-replay-v1",
      "model_key": "lfm3-1.7b",
      "runs": [
       "grading:chat-lfm3-1.7b-seed0-replay-v1",
       "grading:chat-lfm3-1.7b-seed1-r2-v1",
       "grading:chat-lfm3-1.7b-seed2-r2-v1"
      ],
      "aggregation": "mean of 3 seeds; latency over all requests of the three runs",
      "latency_s_median": 5.568,
      "latency_s_p90": 16.417,
      "requests_in_flight": 30,
      "requests_measured": 6207,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 268.3,
      "recall_reference": 0.9176,
      "recall_other": 0.6332,
      "recall_none": 0.83,
      "wrong_answer_recall": 0.6332,
      "false_alarm_rate": 0.041,
      "invalid": null,
      "invalid_per_seed": [
       2,
       1,
       1
      ],
      "probabilities": false
     },
     {
      "run": "grading:chat-minicpm5-2b-seed0-r2-v1",
      "model_key": "minicpm5-2b",
      "runs": [
       "grading:chat-minicpm5-2b-seed0-r2-v1",
       "grading:chat-minicpm5-2b-seed1-r2-v1",
       "grading:chat-minicpm5-2b-seed2-r2-v1"
      ],
      "aggregation": "mean of 3 seeds; latency over all requests of the three runs",
      "latency_s_median": 4.997,
      "latency_s_p90": 8.723,
      "requests_in_flight": 30,
      "requests_measured": 6207,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 196.7,
      "recall_reference": 0.8984,
      "recall_other": 0.7741,
      "recall_none": 0.8542,
      "wrong_answer_recall": 0.7741,
      "false_alarm_rate": 0.0646,
      "invalid": null,
      "invalid_per_seed": [
       3,
       0,
       1
      ],
      "probabilities": false
     },
     {
      "run": "grading:chat-gemma-4-12b-r2-v1",
      "model_key": "gemma-4-12b",
      "latency_s_median": 5.121,
      "latency_s_p90": 15.745,
      "requests_in_flight": 64,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 120.9,
      "recall_reference": 0.9735,
      "recall_other": 0.5198,
      "recall_none": 0.8321,
      "wrong_answer_recall": 0.5198,
      "false_alarm_rate": 0.0264,
      "invalid": 0,
      "probabilities": false
     },
     {
      "run": "grading:chat-lfm3-1.7b-nothink-r2-v1",
      "model_key": "lfm3-1.7b-nothink",
      "runs": [
       "grading:chat-lfm3-1.7b-nothink-r2-v1",
       "grading:chat-lfm3-1.7b-nothink-seed1-r2-v1",
       "grading:chat-lfm3-1.7b-nothink-seed2-r2-v1"
      ],
      "aggregation": "mean of 3 seeds; latency over all requests of the three runs",
      "latency_s_median": 7.618,
      "latency_s_p90": 22.257,
      "requests_in_flight": 64,
      "requests_measured": 6207,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 179.9,
      "recall_reference": 0.9211,
      "recall_other": 0.6261,
      "recall_none": 0.8133,
      "wrong_answer_recall": 0.6261,
      "false_alarm_rate": 0.0387,
      "invalid": null,
      "invalid_per_seed": [
       2,
       2,
       1
      ],
      "probabilities": false
     },
     {
      "run": "grading:chat-gemma-4-e2b-think-r2-v1",
      "model_key": "gemma-4-e2b-think",
      "latency_s_median": 4.022,
      "latency_s_p90": 5.851,
      "requests_in_flight": 64,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 69.4,
      "recall_reference": 0.896,
      "recall_other": 0.4954,
      "recall_none": 0.8321,
      "wrong_answer_recall": 0.4954,
      "false_alarm_rate": 0.0511,
      "invalid": 0,
      "probabilities": false
     },
     {
      "run": "grading:chat-qwen3.5-2b-r2-v1",
      "model_key": "qwen3.5-2b",
      "latency_s_median": 36.808,
      "latency_s_p90": 66.933,
      "requests_in_flight": 64,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 635.8,
      "recall_reference": 0.8652,
      "recall_other": 0.5532,
      "recall_none": 0.8308,
      "wrong_answer_recall": 0.5532,
      "false_alarm_rate": 0.0431,
      "invalid": 78,
      "probabilities": false
     },
     {
      "run": "grading:chat-minicpm5-2b-nothink-r2-v1",
      "model_key": "minicpm5-2b-nothink",
      "runs": [
       "grading:chat-minicpm5-2b-nothink-r2-v1",
       "grading:chat-minicpm5-2b-nothink-seed1-r2-v1",
       "grading:chat-minicpm5-2b-nothink-seed2-r2-v1"
      ],
      "aggregation": "mean of 3 seeds; latency over all requests of the three runs",
      "latency_s_median": 0.858,
      "latency_s_p90": 3.485,
      "requests_in_flight": 64,
      "requests_measured": 6207,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 51.3,
      "recall_reference": 0.8793,
      "recall_other": 0.2877,
      "recall_none": 0.8655,
      "wrong_answer_recall": 0.2877,
      "false_alarm_rate": 0.0272,
      "invalid": null,
      "invalid_per_seed": [
       6,
       5,
       4
      ],
      "probabilities": false
     },
     {
      "run": "grading:chat-gemma-4-12b-think-r2-v1",
      "model_key": "gemma-4-12b-think",
      "latency_s_median": 32.568,
      "latency_s_p90": 1640.171,
      "requests_in_flight": 64,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 4327.1,
      "recall_reference": 0.9342,
      "recall_other": 0.3799,
      "recall_none": 0.7744,
      "wrong_answer_recall": 0.3799,
      "false_alarm_rate": 0.0069,
      "invalid": 296,
      "probabilities": false
     },
     {
      "run": "grading:chat-qwen3.5-4b-r2-v1",
      "model_key": "qwen3.5-4b",
      "latency_s_median": 37.828,
      "latency_s_p90": 96.135,
      "requests_in_flight": 64,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 740.9,
      "recall_reference": 0.8769,
      "recall_other": 0.7325,
      "recall_none": 0.6717,
      "wrong_answer_recall": 0.7325,
      "false_alarm_rate": 0.0615,
      "invalid": 179,
      "probabilities": false
     },
     {
      "run": "grading:chat-gemma-4-e2b-r2-v1",
      "model_key": "gemma-4-e2b",
      "latency_s_median": 0.194,
      "latency_s_p90": 3.314,
      "requests_in_flight": 64,
      "requests_measured": 2069,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 18.3,
      "recall_reference": 0.9469,
      "recall_other": 0.0608,
      "recall_none": 0.6679,
      "wrong_answer_recall": 0.0608,
      "false_alarm_rate": 0.0132,
      "invalid": 0,
      "probabilities": false
     },
     {
      "run": "grading:chat-gemma-4-e2b-sampled-seed0-v1",
      "model_key": "gemma-4-e2b-sampled",
      "runs": [
       "grading:chat-gemma-4-e2b-sampled-seed0-v1",
       "grading:chat-gemma-4-e2b-sampled-seed1-v1",
       "grading:chat-gemma-4-e2b-sampled-seed2-v1"
      ],
      "aggregation": "mean of 3 seeds; latency over all requests of the three runs",
      "latency_s_median": 0.176,
      "latency_s_p90": 5.421,
      "requests_in_flight": 64,
      "requests_measured": 6207,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 33.0,
      "recall_reference": 0.9427,
      "recall_other": 0.077,
      "recall_none": 0.6734,
      "wrong_answer_recall": 0.077,
      "false_alarm_rate": 0.0155,
      "invalid": null,
      "invalid_per_seed": [
       1,
       0,
       1
      ],
      "probabilities": false
     },
     {
      "run": "grading:chat-gemma-4-e4b-think-sampled-seed0-v1",
      "model_key": "gemma-4-e4b-think-sampled",
      "runs": [
       "grading:chat-gemma-4-e4b-think-sampled-seed0-v1",
       "grading:chat-gemma-4-e4b-think-sampled-seed1-v1",
       "grading:chat-gemma-4-e4b-think-sampled-seed2-v1"
      ],
      "aggregation": "mean of 3 seeds; latency over all requests of the three runs",
      "latency_s_median": 6.135,
      "latency_s_p90": 9.7,
      "requests_in_flight": 30,
      "requests_measured": 6207,
      "api_cost_per_1000_usd_list_price": null,
      "price_usd_per_million_input": null,
      "price_source": null,
      "price_date": null,
      "gpu_seconds_per_1000": 233.6,
      "recall_reference": 0.9105,
      "recall_other": 0.6434,
      "recall_none": 0.8225,
      "wrong_answer_recall": 0.6434,
      "false_alarm_rate": 0.0427,
      "invalid": null,
      "invalid_per_seed": [
       34,
       31,
       30
      ],
      "probabilities": false
     }
    ]
   },
   "sha256": "7c77b40233aac20f19a91807abc3578a31a010247979bca0711b260cdb87391e"
  }
 }
}
