{
 "bench": "math",
 "title": "Math",
 "url": "https://zekiworks.github.io/analysis/logic/math/",
 "updated": "2026-11-16T22:07:58Z",
 "score": {
  "zero": "chance: 1 of 5 options right on every question (accuracy 20%)",
  "hundred": "every one of the 2,163 Math questions right (accuracy 100%)"
 },
 "models": [
  {
   "repo": null,
   "commit": null,
   "model_key": "claude-opus-5.5",
   "name": "Claude Opus 5.5",
   "display_name": "Claude Opus 5.5",
   "tag": "chat",
   "class_label": "Closed · Flagship",
   "base_model": null,
   "setup_label": "Claude Opus 5.5",
   "maker": "Anthropic",
   "kind": "chat",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 99.711049468331,
   "raw": {
    "metric": "accuracy",
    "value": 0.9976883957466481,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:claude-opus-5.5-sub_en",
    "math:claude-opus-5.5-sub_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "qwen4-max",
   "name": "Qwen4-Max",
   "display_name": "Qwen4-Max",
   "tag": "high · chat",
   "class_label": "Closed · Undisclosed",
   "base_model": null,
   "setup_label": "Qwen4-Max",
   "maker": "Alibaba",
   "kind": "chat",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 99.711049468331,
   "raw": {
    "metric": "accuracy",
    "value": 0.9976883957466481,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:qwen4-max_en",
    "math:qwen4-max_en_dontknow"
   ],
   "notes": "hosted API via OpenRouter",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "claude-fable-5.1",
   "name": "Claude Fable 5.1",
   "display_name": "Claude Fable 5.1",
   "tag": "chat",
   "class_label": "Closed · Flagship",
   "base_model": null,
   "setup_label": "Claude Fable 5.1",
   "maker": "Anthropic",
   "kind": "chat",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 99.65325936199723,
   "raw": {
    "metric": "accuracy",
    "value": 0.9972260748959778,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:claude-fable-5.1-sub_en",
    "math:claude-fable-5.1-sub_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "claude-sonnet-5.5",
   "name": "Claude Sonnet 5.5",
   "display_name": "Claude Sonnet 5.5",
   "tag": "chat",
   "class_label": "Closed · Workhorse",
   "base_model": null,
   "setup_label": "Claude Sonnet 5.5",
   "maker": "Anthropic",
   "kind": "chat",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 99.59546925566343,
   "raw": {
    "metric": "accuracy",
    "value": 0.9967637540453075,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:claude-sonnet-5.5-sub_en",
    "math:claude-sonnet-5.5-sub_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "deepseek-v4.1-flash-1017",
   "name": "DeepSeek-V4.1-Flash",
   "display_name": "DeepSeek-V4.1-Flash",
   "tag": "high · chat · 1017",
   "class_label": "Open · 552B MoE · 16B active",
   "base_model": null,
   "setup_label": "DeepSeek-V4.1-Flash 1017",
   "maker": "DeepSeek",
   "kind": "chat",
   "params_total": 552.0,
   "params_active": 16.0,
   "quantization": null,
   "score_0_100": 99.53767914932962,
   "raw": {
    "metric": "accuracy",
    "value": 0.996301433194637,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:deepseek-v4.1-flash-1017_en",
    "math:deepseek-v4.1-flash-1017_en_dontknow"
   ],
   "notes": "hosted API via OpenRouter",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "gpt-6-astra",
   "name": "GPT-6 Astra",
   "display_name": "GPT-6 Astra",
   "tag": "chat",
   "class_label": "Closed · Undisclosed",
   "base_model": null,
   "setup_label": "GPT-6 Astra",
   "maker": "OpenAI",
   "kind": "chat",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 99.53767914932962,
   "raw": {
    "metric": "accuracy",
    "value": 0.996301433194637,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:gpt-6-astra-sub_en",
    "math:gpt-6-astra-sub_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "gpt-6.1-sol",
   "name": "GPT-6.1 Sol",
   "display_name": "GPT-6.1 Sol",
   "tag": "chat",
   "class_label": "Closed · Undisclosed",
   "base_model": null,
   "setup_label": "GPT-6.1 Sol",
   "maker": "OpenAI",
   "kind": "chat",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 99.53767914932962,
   "raw": {
    "metric": "accuracy",
    "value": 0.996301433194637,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:gpt-6.1-sol-sub_en",
    "math:gpt-6.1-sol-sub_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "inkling-small",
   "name": "Inkling-Small",
   "display_name": "Inkling-Small",
   "tag": "high · chat · Jul 2026",
   "class_label": "Open · 276B MoE · 12B active",
   "base_model": null,
   "setup_label": "Inkling Small",
   "maker": "Thinking Machines",
   "kind": "chat",
   "params_total": 276.0,
   "params_active": 12.0,
   "quantization": null,
   "score_0_100": 99.53767914932962,
   "raw": {
    "metric": "accuracy",
    "value": 0.996301433194637,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:inkling-small-64k_en",
    "math:inkling-small-64k_en_dontknow"
   ],
   "notes": "hosted API via OpenRouter",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "gemini-4-carbon",
   "name": "Gemini 4 Carbon",
   "display_name": "Gemini 4 Carbon",
   "tag": "chat",
   "class_label": "Closed · Undisclosed",
   "base_model": null,
   "setup_label": "Gemini 4 Carbon (Antigravity subscription)",
   "maker": "Google",
   "kind": "chat",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 99.47988904299584,
   "raw": {
    "metric": "accuracy",
    "value": 0.9958391123439667,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:gemini-4-carbon-sub_en",
    "math:gemini-4-carbon-sub_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "gemini-4-hydrogen",
   "name": "Gemini 4 Hydrogen",
   "display_name": "Gemini 4 Hydrogen",
   "tag": "chat",
   "class_label": "",
   "base_model": null,
   "setup_label": "Gemini 4 Hydrogen (Antigravity subscription)",
   "maker": "Google",
   "kind": "chat",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 99.47988904299584,
   "raw": {
    "metric": "accuracy",
    "value": 0.9958391123439667,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:gemini-4-hydrogen-sub_en",
    "math:gemini-4-hydrogen-sub_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "muse-ember",
   "name": "Muse Ember",
   "display_name": "Muse Ember",
   "tag": "max · chat",
   "class_label": "Closed · Efficient",
   "base_model": null,
   "setup_label": "Muse Ember",
   "maker": "Meta",
   "kind": "chat",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 99.47988904299584,
   "raw": {
    "metric": "accuracy",
    "value": 0.9958391123439667,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:muse-ember_en",
    "math:muse-ember_en_dontknow"
   ],
   "notes": "hosted API via OpenRouter",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "grok-4.7",
   "name": "Grok 4.7",
   "display_name": "Grok 4.7",
   "tag": "chat",
   "class_label": "Closed · Undisclosed",
   "base_model": null,
   "setup_label": "Grok 4.7",
   "maker": "xAI",
   "kind": "chat",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 99.42209893666204,
   "raw": {
    "metric": "accuracy",
    "value": 0.9953767914932964,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:grok-4.7-sub_en",
    "math:grok-4.7-sub_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "inkling-2-small",
   "name": "Inkling 2 Small",
   "display_name": "Inkling 2 Small",
   "tag": "high · chat",
   "class_label": "Open · 594B MoE · 17B active",
   "base_model": null,
   "setup_label": "Inkling 2 Small",
   "maker": "Thinking Machines",
   "kind": "chat",
   "params_total": 594.0,
   "params_active": 17.0,
   "quantization": null,
   "score_0_100": 99.42209893666204,
   "raw": {
    "metric": "accuracy",
    "value": 0.9953767914932964,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:inkling-2-small_en",
    "math:inkling-2-small_en_dontknow"
   ],
   "notes": "hosted API via OpenRouter",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "qwen3.8-flash",
   "name": "Qwen3.8 Flash",
   "display_name": "Qwen3.8 Flash",
   "tag": "off · chat",
   "class_label": "Closed · Undisclosed",
   "base_model": null,
   "setup_label": "Qwen3.8 Flash",
   "maker": "Alibaba",
   "kind": "chat",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 99.36430883032823,
   "raw": {
    "metric": "accuracy",
    "value": 0.994914470642626,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:qwen3.8-flash_en",
    "math:qwen3.8-flash_en_dontknow"
   ],
   "notes": "hosted API via OpenRouter",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "mimo-v2.6-flash",
   "name": "MiMo-V2.6-Flash",
   "display_name": "MiMo-V2.6-Flash",
   "tag": "on · chat",
   "class_label": "Open · 309B MoE · 15B active",
   "base_model": null,
   "setup_label": "MiMo V2.6 Flash",
   "maker": "Xiaomi",
   "kind": "chat",
   "params_total": 309.0,
   "params_active": 15.0,
   "quantization": null,
   "score_0_100": 99.30651872399446,
   "raw": {
    "metric": "accuracy",
    "value": 0.9944521497919556,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:mimo-v2.6-flash-64k_en",
    "math:mimo-v2.6-flash-64k_en_dontknow"
   ],
   "notes": "hosted API via OpenRouter",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": "Qwen/Qwen3.5-4B",
   "commit": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",
   "model_key": "qwen3.5-4b",
   "name": "Qwen3.5-4B",
   "display_name": "Qwen3.5-4B",
   "tag": "on · chat",
   "class_label": "Open · 4.7B",
   "base_model": null,
   "setup_label": "Qwen3.5-4B (chat, thinking, on our GPU)",
   "maker": "Alibaba",
   "kind": "chat",
   "params_total": 4.66,
   "params_active": 4.66,
   "quantization": "BF16",
   "score_0_100": 99.30651872399446,
   "raw": {
    "metric": "accuracy",
    "value": 0.9944521497919556,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:qwen3.5-4b_en",
    "math:qwen3.5-4b_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: Qwen3.5-4B (off · chat)",
   "gpu_hours": 1.1041666666666667,
   "gpu_cost_usd": 2.749375
  },
  {
   "repo": "google/gemma-4-31B-it",
   "commit": "842da3794eaa0b77d5f08bae87a17459d91ff475",
   "model_key": "gemma-4-31b",
   "name": "Gemma 4 31B",
   "display_name": "Gemma 4 31B",
   "tag": "chat",
   "class_label": "Open · 31B",
   "base_model": null,
   "setup_label": "Gemma 4 31B (chat, on our GPU)",
   "maker": "Google",
   "kind": "chat",
   "params_total": 30.7,
   "params_active": 30.7,
   "quantization": "FP8",
   "score_0_100": 99.24872861766065,
   "raw": {
    "metric": "accuracy",
    "value": 0.9939898289412853,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:gemma-4-31b-it_en_t0",
    "math:gemma-4-31b-it_en_t0_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); BF16 weights quantized to FP8 by vLLM at load",
   "gpu_hours": 0.46111111111111114,
   "gpu_cost_usd": 1.1481666666666668
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "deepseek-v4-flash-dspark",
   "name": "DeepSeek-V4-Flash",
   "display_name": "DeepSeek-V4-Flash",
   "tag": "chat",
   "class_label": "Open · 285B MoE · 13B active",
   "base_model": null,
   "setup_label": "deepseek-v4-flash-dspark",
   "maker": "DeepSeek",
   "kind": "chat",
   "params_total": 285.0,
   "params_active": 13.0,
   "quantization": null,
   "score_0_100": 99.13314840499307,
   "raw": {
    "metric": "accuracy",
    "value": 0.9930651872399445,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:deepseek-v4-flash_en_t0"
   ],
   "notes": "on our NVIDIA DGX Spark (GB10); no complete 'I don't know' run",
   "gpu_hours": 2.9963888888888888,
   "gpu_cost_usd": 0.6376849694375101
  },
  {
   "repo": "openbmb/MiniCPM5-2B",
   "commit": "42433e90dad95123ab87729fd7011e7110629556",
   "model_key": "minicpm5-2b",
   "name": "MiniCPM5-2B",
   "display_name": "MiniCPM5-2B",
   "tag": "on · chat",
   "class_label": "Open · 2.5B",
   "base_model": null,
   "setup_label": "MiniCPM5-2B (chat, thinking, on our GPU)",
   "maker": "OpenBMB",
   "kind": "chat",
   "params_total": 2.52,
   "params_active": 2.52,
   "quantization": "BF16",
   "score_0_100": 99.0368315611034,
   "raw": {
    "metric": "accuracy",
    "value": 0.9922946524888273,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:minicpm5-2b_en",
    "math:minicpm5-2b_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: MiniCPM5-2B (off · chat); accuracy is the mean of 3 seed runs (min 0.9917, max 0.9926): seed 0 minicpm5-2b_en 2147/2163, seed 1 minicpm5-2b-seed1_en 2145/2163, seed 2 minicpm5-2b-seed2_en 2147/2163",
   "gpu_hours": 0.3605555555555556,
   "gpu_cost_usd": 0.8977833333333335
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "gpt-6.1-terra",
   "name": "GPT-6.1 Terra",
   "display_name": "GPT-6.1 Terra",
   "tag": "chat",
   "class_label": "Closed · Undisclosed",
   "base_model": null,
   "setup_label": "GPT-6.1 Terra",
   "maker": "OpenAI",
   "kind": "chat",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 99.01756819232546,
   "raw": {
    "metric": "accuracy",
    "value": 0.9921405455386038,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:gpt-6.1-terra-sub_en",
    "math:gpt-6.1-terra-sub_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": "ibm-granite/granite-4.2-3b",
   "commit": "4e9c275e232f10211ec88644d106a1f6a1b60e06",
   "model_key": "granite-4.2-3b",
   "name": "Granite-4.2-3B",
   "display_name": "Granite-4.2-3B",
   "tag": "high · chat",
   "class_label": "Open · 3.0B",
   "base_model": "Granite-4.1-3B-Base",
   "setup_label": "Granite-4.2-3B (chat, thinking, on our GPU)",
   "maker": "IBM",
   "kind": "chat",
   "params_total": 3.0,
   "params_active": 3.0,
   "quantization": "BF16",
   "score_0_100": 99.01756819232546,
   "raw": {
    "metric": "accuracy",
    "value": 0.9921405455386038,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:granite-4.2-3b-64k_en",
    "math:granite-4.2-3b-64k_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.6783333333333333,
   "gpu_cost_usd": 1.6890500000000002
  },
  {
   "repo": "Qwen/Qwen3.8-27B",
   "commit": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
   "model_key": "qwen3.8-27b",
   "name": "Qwen3.8-27B",
   "display_name": "Qwen3.8-27B",
   "tag": "low · chat",
   "class_label": "Open · 27B",
   "base_model": null,
   "setup_label": "Qwen3.8-27B (chat, reasoning low, on our GPU)",
   "maker": "Alibaba",
   "kind": "chat",
   "params_total": 27.0,
   "params_active": 27.0,
   "quantization": "BF16",
   "score_0_100": 99.01756819232546,
   "raw": {
    "metric": "accuracy",
    "value": 0.9921405455386038,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:qwen3.8-27b-bf16_en_low",
    "math:qwen3.8-27b-bf16_en_low_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.4602777777777778,
   "gpu_cost_usd": 1.1460916666666667
  },
  {
   "repo": "LiquidAI/LFM2.5-24B-A2B",
   "commit": "1769516ac588c4c558089c94f27d96b0dc67b183",
   "model_key": "lfm2.5-24b-a2b",
   "name": "LFM2.5-24B-A2B",
   "display_name": "LFM2.5-24B-A2B",
   "tag": "on · chat",
   "class_label": "Open · 24B MoE · 2.5B active",
   "base_model": null,
   "setup_label": "LFM2.5-24B-A2B (chat, thinking, on our GPU)",
   "maker": "Liquid AI",
   "kind": "chat",
   "params_total": 24.0,
   "params_active": 2.45,
   "quantization": "BF16",
   "score_0_100": 98.7864077669903,
   "raw": {
    "metric": "accuracy",
    "value": 0.9902912621359223,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:lfm2.5-24b-a2b_en",
    "math:lfm2.5-24b-a2b_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.8158333333333333,
   "gpu_cost_usd": 2.0314250000000005
  },
  {
   "repo": "LiquidAI/LFM3-1.7B",
   "commit": "0dc4268b561499c648c7ba7c9466bf3d8d01517a",
   "model_key": "lfm3-1.7b",
   "name": "LFM3-1.7B",
   "display_name": "LFM3-1.7B",
   "tag": "on · chat",
   "class_label": "Open · 1.7B",
   "base_model": null,
   "setup_label": "LFM3-1.7B (chat, thinking, on our GPU)",
   "maker": null,
   "kind": "chat",
   "params_total": 1.73,
   "params_active": 1.73,
   "quantization": "BF16",
   "score_0_100": 98.67082755432268,
   "raw": {
    "metric": "accuracy",
    "value": 0.9893666204345816,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:lfm3-1.7b-vllm_en",
    "math:lfm3-1.7b_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: LFM3-1.7B (off · chat), LFM3-1.7B (off · chat · NVIDIA RTX PRO 6000 Blackwell); accuracy is the mean of 3 seed runs (min 0.9884, max 0.9908): seed 0 lfm3-1.7b-vllm_en 2139/2163, seed 1 lfm3-1.7b-vllm-seed1_en 2143/2163, seed 2 lfm3-1.7b-vllm-seed2_en 2138/2163",
   "gpu_hours": 0.11277777777777778,
   "gpu_cost_usd": 0.28081666666666666
  },
  {
   "repo": "google/gemma-4-12B-it",
   "commit": "43c9d014b231474fa7212dbb48a762566c2b255d",
   "model_key": "gemma-4-12b",
   "name": "Gemma 4 12B Unified",
   "display_name": "Gemma 4 12B Unified",
   "tag": "chat",
   "class_label": "Open · 12B",
   "base_model": null,
   "setup_label": "Gemma 4 12B (chat, on our GPU)",
   "maker": "Google",
   "kind": "chat",
   "params_total": 11.95,
   "params_active": 11.95,
   "quantization": "BF16",
   "score_0_100": 98.5552473416551,
   "raw": {
    "metric": "accuracy",
    "value": 0.9884419787332409,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:gemma-4-12b-it_en",
    "math:gemma-4-12b-it_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.4058333333333333,
   "gpu_cost_usd": 1.0105250000000001
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "glm-5.3-flash",
   "name": "GLM-5.3-Flash",
   "display_name": "GLM-5.3-Flash",
   "tag": "high · chat",
   "class_label": "Open · 320B MoE · 18B active",
   "base_model": null,
   "setup_label": "GLM-5.3-Flash",
   "maker": "Z.ai",
   "kind": "chat",
   "params_total": 320.0,
   "params_active": 18.0,
   "quantization": null,
   "score_0_100": 98.49745723532129,
   "raw": {
    "metric": "accuracy",
    "value": 0.9879796578825705,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:glm-5.3-flash_en",
    "math:glm-5.3-flash_en_dontknow"
   ],
   "notes": "hosted API via OpenRouter",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "minimax-m3.1-flash",
   "name": "MiniMax-M3.1-Flash",
   "display_name": "MiniMax-M3.1-Flash",
   "tag": "high · chat",
   "class_label": "Open · 428B MoE · 23B active",
   "base_model": null,
   "setup_label": "MiniMax-M3.1-Flash",
   "maker": "MiniMax",
   "kind": "chat",
   "params_total": 428.0,
   "params_active": 23.0,
   "quantization": null,
   "score_0_100": 97.80397595931575,
   "raw": {
    "metric": "accuracy",
    "value": 0.9824318076745261,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:minimax-m3.1-flash_en",
    "math:minimax-m3.1-flash_en_dontknow"
   ],
   "notes": "hosted API via OpenRouter",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": "google/gemma-4-E4B-it",
   "commit": "4ecc2ba63bcac23ba4a767fc87025b1450f84c9c",
   "model_key": "gemma-4-e4b",
   "name": "Gemma 4 E4B",
   "display_name": "Gemma 4 E4B",
   "tag": "chat",
   "class_label": "Open · 8.0B",
   "base_model": null,
   "setup_label": "Gemma 4 E4B (chat, on our GPU)",
   "maker": "Google",
   "kind": "chat",
   "params_total": 8.0,
   "params_active": 8.0,
   "quantization": "BF16",
   "score_0_100": 97.63060564031436,
   "raw": {
    "metric": "accuracy",
    "value": 0.981044845122515,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:gemma-4-e4b-it_en",
    "math:gemma-4-e4b-it_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: Gemma 4 E4B (on · chat), Gemma 4 E4B (on · chat · 32k cap), Gemma 4 E4B (on · chat · 32k cap · sampled); 4.5B effective parameters",
   "gpu_hours": 0.23555555555555555,
   "gpu_cost_usd": 0.5865333333333334
  },
  {
   "repo": "google/gemma-4-E2B-it",
   "commit": "4adf15748de9e1a0097b6274cee9c15836c41c37",
   "model_key": "gemma-4-e2b",
   "name": "Gemma 4 E2B",
   "display_name": "Gemma 4 E2B",
   "tag": "chat",
   "class_label": "Open · 5.1B",
   "base_model": null,
   "setup_label": "Gemma 4 E2B (chat, on our GPU)",
   "maker": "Google",
   "kind": "chat",
   "params_total": 5.1,
   "params_active": 5.1,
   "quantization": "BF16",
   "score_0_100": 95.9546925566343,
   "raw": {
    "metric": "accuracy",
    "value": 0.9676375404530745,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:gemma-4-e2b-it_en",
    "math:gemma-4-e2b-it_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: Gemma 4 E2B (on · chat), Gemma 4 E2B (on · chat · 32k cap), Gemma 4 E2B (on · chat · 32k cap · sampled); 2.3B effective parameters",
   "gpu_hours": 0.18083333333333335,
   "gpu_cost_usd": 0.4502750000000001
  },
  {
   "repo": "LiquidAI/LFM2.5-8B-A1B",
   "commit": "dd83f5cb02aa2938b66bba9a9998d38d0517c1f3",
   "model_key": "lfm2.5-8b-a1b",
   "name": "LFM2.5-8B-A1B",
   "display_name": "LFM2.5-8B-A1B",
   "tag": "on · chat",
   "class_label": "Open · 8.3B MoE · 1.5B active",
   "base_model": null,
   "setup_label": "LFM2.5-8B-A1B (chat, thinking, on our GPU)",
   "maker": "Liquid AI",
   "kind": "chat",
   "params_total": 8.3,
   "params_active": 1.5,
   "quantization": "BF16",
   "score_0_100": 95.31900138696255,
   "raw": {
    "metric": "accuracy",
    "value": 0.9625520110957004,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:lfm2.5-8b-a1b_en",
    "math:lfm2.5-8b-a1b_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.41888888888888887,
   "gpu_cost_usd": 1.0430333333333335
  },
  {
   "repo": "Qwen/Qwen3-14B",
   "commit": "40c069824f4251a91eefaf281ebe4c544efd3e18",
   "model_key": "qwen3-14b",
   "name": "Qwen3-14B",
   "display_name": "Qwen3-14B",
   "tag": "off · chat",
   "class_label": "Open · 15B",
   "base_model": null,
   "setup_label": "Qwen3-14B (chat, reasoning off, on our GPU)",
   "maker": "Alibaba",
   "kind": "chat",
   "params_total": 14.8,
   "params_active": 14.8,
   "quantization": "BF16",
   "score_0_100": 94.7411003236246,
   "raw": {
    "metric": "accuracy",
    "value": 0.9579288025889967,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:qwen3-14b_en",
    "math:qwen3-14b_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.30833333333333335,
   "gpu_cost_usd": 0.76775
  },
  {
   "repo": "Qwen/Qwen3.5-2B",
   "commit": "15852e8c16360a2fea060d615a32b45270f8a8fc",
   "model_key": "qwen3.5-2b",
   "name": "Qwen3.5-2B",
   "display_name": "Qwen3.5-2B",
   "tag": "off · chat",
   "class_label": "Open · 2.0B",
   "base_model": null,
   "setup_label": "Qwen3.5-2B (chat, thinking off, on our GPU)",
   "maker": "Alibaba",
   "kind": "chat",
   "params_total": 2.0,
   "params_active": 2.0,
   "quantization": "BF16",
   "score_0_100": 93.29634766527968,
   "raw": {
    "metric": "accuracy",
    "value": 0.9463707813222376,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:qwen3.5-2b-nothink_en",
    "math:qwen3.5-2b-nothink_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: Qwen3.5-2B (on · chat)",
   "gpu_hours": 0.31555555555555553,
   "gpu_cost_usd": 0.7857333333333334
  },
  {
   "repo": "LiquidAI/LFM2.5-2.6B",
   "commit": "7141b56cd4650af95c7c7b86983443aed050c96a",
   "model_key": "lfm2.5-2.6b",
   "name": "LFM2.5-2.6B",
   "display_name": "LFM2.5-2.6B",
   "tag": "on · chat",
   "class_label": "Open · 2.7B",
   "base_model": null,
   "setup_label": "LFM2.5-2.6B (chat, thinking, on our GPU)",
   "maker": "Liquid AI",
   "kind": "chat",
   "params_total": 2.69,
   "params_active": 2.69,
   "quantization": "BF16",
   "score_0_100": 92.48728617660656,
   "raw": {
    "metric": "accuracy",
    "value": 0.9398982894128525,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:lfm2.5-2.6b_en",
    "math:lfm2.5-2.6b_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.26055555555555554,
   "gpu_cost_usd": 0.6487833333333334
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "gpt-6-luna",
   "name": "GPT-6 Luna",
   "display_name": "GPT-6 Luna",
   "tag": "chat",
   "class_label": "Closed · Efficient",
   "base_model": null,
   "setup_label": "GPT-6 Luna (low)",
   "maker": "OpenAI",
   "kind": "chat",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 92.08275543226998,
   "raw": {
    "metric": "accuracy",
    "value": 0.93666204345816,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:gpt-6-luna-low-sub_en",
    "math:gpt-6-luna-low-sub_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": "ecloudtech/Erk-14B",
   "commit": "8584b14a829d5cc803bc227f44e9012e36e8ad04",
   "model_key": "erk-14b",
   "name": "Erk-14B",
   "display_name": "Erk-14B",
   "tag": "off · chat",
   "class_label": "Open · 15B",
   "base_model": null,
   "setup_label": "Erk-14B (chat, reasoning off, on our GPU)",
   "maker": "eCloud Technology",
   "kind": "chat",
   "params_total": 14.77,
   "params_active": 14.77,
   "quantization": "BF16",
   "score_0_100": 88.44197873324086,
   "raw": {
    "metric": "accuracy",
    "value": 0.907535829865927,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:erk-14b_en",
    "math:erk-14b_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.2311111111111111,
   "gpu_cost_usd": 0.5754666666666668
  },
  {
   "repo": "KRAFTON/Raon-1-nano",
   "commit": "fcf2d1d55e84e2f9fbe7b5d724dcfc4d861cfcee",
   "model_key": "raon-1-nano",
   "name": "Raon-1 nano",
   "display_name": "Raon-1 nano",
   "tag": "off · chat",
   "class_label": "Open · 3.1B",
   "base_model": "Raon-1 mini preview (MoE-to-Dense pruning and distillation)",
   "setup_label": "Raon-1 nano (chat, thinking off, KRAFTON vLLM fork, on our GPU)",
   "maker": "KRAFTON",
   "kind": "chat",
   "params_total": 3.1,
   "params_active": 3.1,
   "quantization": null,
   "score_0_100": 86.93943596856218,
   "raw": {
    "metric": "accuracy",
    "value": 0.8955154877484974,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "math:raon-1-nano-nothink_en",
    "math:raon-1-nano-nothink_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); other setups of this model under its row, not in this entry: Raon-1 nano (on · chat); 1.5B effective parameters",
   "gpu_hours": 1.6511111111111112,
   "gpu_cost_usd": 4.111266666666667
  },
  {
   "repo": "MachinePulseAI/karpo-decision-31B",
   "commit": "d69cdd57eab23271cbe65236ca8a00d80d7bb8c7",
   "model_key": "karpo-decision-31b",
   "name": "Karpo Decision 31B",
   "display_name": "Karpo Decision 31B",
   "tag": "",
   "class_label": "Open · 31B",
   "base_model": null,
   "setup_label": "karpo-decision-31B",
   "maker": "MachinePulse",
   "kind": "decision",
   "params_total": 31.0,
   "params_active": 31.0,
   "quantization": "BF16",
   "score_0_100": 76.94174757281552,
   "raw": {
    "metric": "accuracy",
    "value": 0.8155339805825242,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.012315314621813527,
   "improvement_points": null,
   "runs": [
    "math:karpo-decision-31b_en",
    "math:karpo-decision-31b_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.09777777777777778,
   "gpu_cost_usd": 0.24346666666666666
  },
  {
   "repo": "PelaAI/KnowLine-31B-Gen2",
   "commit": "fd99a9ed2255d8a67dd47a44cc429e638f611593",
   "model_key": "knowline-31b-gen2",
   "name": "KnowLine-31B-Gen2",
   "display_name": "KnowLine-31B-Gen2",
   "tag": "",
   "class_label": "Open · 31B",
   "base_model": "Gemma 4 31B",
   "setup_label": "KnowLine-31B-Gen2",
   "maker": "PelaAI",
   "kind": "decision",
   "params_total": 31.0,
   "params_active": 31.0,
   "quantization": "BF16",
   "score_0_100": 75.32362459546927,
   "raw": {
    "metric": "accuracy",
    "value": 0.8025889967637541,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.03631832690906286,
   "improvement_points": null,
   "runs": [
    "math:knowline-31b-gen2_en",
    "math:knowline-31b-gen2_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.07222222222222222,
   "gpu_cost_usd": 0.17983333333333335
  },
  {
   "repo": "decisionmodels/REAX-1.3",
   "commit": "fa4bf7ee3bb56a8bc421dad090631be1c5845c2d",
   "model_key": "reax-1.3",
   "name": "REAX 1.3",
   "display_name": "REAX 1.3",
   "tag": "",
   "class_label": "Open · 57B",
   "base_model": null,
   "setup_label": "REAX 1.3",
   "maker": "REAX",
   "kind": "decision",
   "params_total": 57.36,
   "params_active": 57.36,
   "quantization": "FP8",
   "score_0_100": 72.49190938511326,
   "raw": {
    "metric": "accuracy",
    "value": 0.7799352750809061,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.07101455985854778,
   "improvement_points": null,
   "runs": [
    "math:reax13_en",
    "math:reax13_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); weights byte-identical to decisionmodels/REAX-1.3 at this commit; served by the REAX engine (github.com/reaxlabs/reax-engine ff8f595)",
   "gpu_hours": 0.12333333333333334,
   "gpu_cost_usd": 0.30710000000000004
  },
  {
   "repo": "PelaAI/KnowLine-31B-Gen1.5",
   "commit": "8e56aa1b236a6dd1064dcfd22ba1e5e360e4c140",
   "model_key": "knowline-31b-gen1.5",
   "name": "KnowLine-31B-Gen1.5",
   "display_name": "KnowLine-31B-Gen1.5",
   "tag": "",
   "class_label": "Open · 31B",
   "base_model": null,
   "setup_label": "KnowLine 31B Gen1.5",
   "maker": "PelaAI",
   "kind": "decision",
   "params_total": 31.0,
   "params_active": 31.0,
   "quantization": "BF16",
   "score_0_100": 71.74063800277392,
   "raw": {
    "metric": "accuracy",
    "value": 0.7739251040221914,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.03555228644547992,
   "improvement_points": null,
   "runs": [
    "math:knowline-31b_en",
    "math:knowline-31b_en_dontknow"
   ],
   "notes": "on our NVIDIA DGX Spark (GB10)",
   "gpu_hours": 0.47638888888888886,
   "gpu_cost_usd": 0.10138404770421153
  },
  {
   "repo": "Poindexter-Labs/Copernicus-31B",
   "commit": "be715259e97a743341f2d7a3e6a6298dab23ca0e",
   "model_key": "copernicus-31b",
   "name": "Copernicus-31B",
   "display_name": "Copernicus-31B",
   "tag": "",
   "class_label": "Open · 31B",
   "base_model": null,
   "setup_label": "Copernicus-31B",
   "maker": "Poindexter Labs",
   "kind": "decision",
   "params_total": 31.0,
   "params_active": 31.0,
   "quantization": "BF16",
   "score_0_100": 71.68284789644014,
   "raw": {
    "metric": "accuracy",
    "value": 0.7734627831715211,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.06453940462900969,
   "improvement_points": null,
   "runs": [
    "math:copernicus-31b_en",
    "math:copernicus-31b_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.07333333333333333,
   "gpu_cost_usd": 0.1826
  },
  {
   "repo": "sthanika-ai/Sieve-31B-v3",
   "commit": "8fcee858f7f20a50bfa3c8f984c22adb3cbb46a2",
   "model_key": "sieve-31b-v3",
   "name": "Sieve 31B v3",
   "display_name": "Sieve 31B v3",
   "tag": "",
   "class_label": "Open · 31B",
   "base_model": null,
   "setup_label": "Sieve 31B v3",
   "maker": "Sthānika AI",
   "kind": "decision",
   "params_total": 31.0,
   "params_active": 31.0,
   "quantization": "BF16",
   "score_0_100": 71.39389736477114,
   "raw": {
    "metric": "accuracy",
    "value": 0.7711511789181692,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.042755524734165495,
   "improvement_points": null,
   "runs": [
    "math:sieve-31b_en",
    "math:sieve-31b_en_dontknow"
   ],
   "notes": "on our NVIDIA DGX Spark (GB10)",
   "gpu_hours": 0.3977777777777778,
   "gpu_cost_usd": 0.0846542019314466
  },
  {
   "repo": "logitshq/Logits-Sigma-1-Duo",
   "commit": "a0c1b6e8242916aafdcf9a5a49c1a952cb56e091",
   "model_key": "logits-sigma-1-duo",
   "name": "Logits Sigma-1 Duo",
   "display_name": "Logits Sigma-1 Duo",
   "tag": "",
   "class_label": "Open · 57B",
   "base_model": null,
   "setup_label": "Logits Sigma-1 Duo",
   "maker": "Logits",
   "kind": "decision",
   "params_total": 57.36,
   "params_active": 57.36,
   "quantization": "FP8",
   "score_0_100": 69.02450300508552,
   "raw": {
    "metric": "accuracy",
    "value": 0.7521960240406842,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.07541269945078859,
   "improvement_points": null,
   "runs": [
    "math:logits-sigma-1-duo_en",
    "math:logits-sigma-1-duo_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.12222222222222222,
   "gpu_cost_usd": 0.30433333333333334
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "liquid-d1-ultra",
   "name": "d1-ultra",
   "display_name": "d1-ultra",
   "tag": "",
   "class_label": "Closed · Efficient",
   "base_model": null,
   "setup_label": "Liquid d1-ultra",
   "maker": "Liquid AI",
   "kind": "decision",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 67.63754045307444,
   "raw": {
    "metric": "accuracy",
    "value": 0.7411003236245954,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.02258235015511772,
   "improvement_points": null,
   "runs": [
    "math:d1-ultra_en",
    "math:d1-ultra_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": "baselabs/mammoth-1",
   "commit": "61cec44c281c693992e3d6ebeb44dd3402dfa186",
   "model_key": "mammoth-1",
   "name": "Mammoth-1",
   "display_name": "Mammoth-1",
   "tag": "",
   "class_label": "Open · 27B",
   "base_model": null,
   "setup_label": "Mammoth-1",
   "maker": "Base Labs",
   "kind": "decision",
   "params_total": 27.0,
   "params_active": 27.0,
   "quantization": "BF16",
   "score_0_100": 67.46417013407304,
   "raw": {
    "metric": "accuracy",
    "value": 0.7397133610725843,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.029765094313453477,
   "improvement_points": null,
   "runs": [
    "math:mammoth-1_en",
    "math:mammoth-1_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.09027777777777778,
   "gpu_cost_usd": 0.2247916666666667
  },
  {
   "repo": "perplexity-ai/pplx-decider-v1.3",
   "commit": "9c2ebb27470d3dec2ad7ab727727e40d142c7213",
   "model_key": "perplexity-decider-27b-v1.3",
   "name": "Perplexity Decider 27B v1.3",
   "display_name": "Perplexity Decider 27B v1.3",
   "tag": "NVIDIA RTX PRO 6000 Blackwell",
   "class_label": "Open · 27B",
   "base_model": null,
   "setup_label": "Perplexity Decider 27B v1.3 (local GPU)",
   "maker": "Perplexity",
   "kind": "decision",
   "params_total": 27.0,
   "params_active": 27.0,
   "quantization": "BF16",
   "score_0_100": 65.61488673139159,
   "raw": {
    "metric": "accuracy",
    "value": 0.7249190938511327,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.04459839344013791,
   "improvement_points": null,
   "runs": [
    "math:decider-v1.3-local_en",
    "math:decider-v1.3-local_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.05722222222222222,
   "gpu_cost_usd": 0.14248333333333335
  },
  {
   "repo": "fastino/GLiDE-27B",
   "commit": "4d32743ae96b74f0116f050d37742bbd43c04e7d",
   "model_key": "glide-27b",
   "name": "GLiDE 27B",
   "display_name": "GLiDE 27B",
   "tag": "",
   "class_label": "Open · 27B",
   "base_model": null,
   "setup_label": "GLiDE 27B (open)",
   "maker": "Fastino",
   "kind": "decision",
   "params_total": 27.0,
   "params_active": 27.0,
   "quantization": "BF16",
   "score_0_100": 64.97919556171982,
   "raw": {
    "metric": "accuracy",
    "value": 0.7198335644937587,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.04077789772290079,
   "improvement_points": null,
   "runs": [
    "math:glide-27b_en",
    "math:glide-27b_en_dontknow"
   ],
   "notes": "on our NVIDIA DGX Spark (GB10)",
   "gpu_hours": 0.2658333333333333,
   "gpu_cost_usd": 0.05657407210083407
  },
  {
   "repo": "vllm-sr/d3-ultra",
   "commit": "52ba9e7a00402f9560c7fba8c039608e7ddee7fb",
   "model_key": "vllm-sr-d3-ultra",
   "name": "d3-ultra",
   "display_name": "d3-ultra",
   "tag": "",
   "class_label": "Open · 26B",
   "base_model": null,
   "setup_label": "vLLM-SR d3-ultra",
   "maker": "vLLM Semantic Router",
   "kind": "decision",
   "params_total": 26.09,
   "params_active": 26.09,
   "quantization": "BF16",
   "score_0_100": 64.40129449838189,
   "raw": {
    "metric": "accuracy",
    "value": 0.7152103559870551,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.08639034384095057,
   "improvement_points": null,
   "runs": [
    "math:d3-ultra_en",
    "math:d3-ultra_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.07694444444444444,
   "gpu_cost_usd": 0.19159166666666666
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "gemini-decision-1",
   "name": "Gemini Decision 1",
   "display_name": "Gemini Decision 1",
   "tag": "20261111",
   "class_label": "Closed · Efficient",
   "base_model": null,
   "setup_label": "Gemini Decision 1",
   "maker": "Google",
   "kind": "decision",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 64.1701340730467,
   "raw": {
    "metric": "accuracy",
    "value": 0.7133610725843735,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.024586229897428504,
   "improvement_points": null,
   "runs": [
    "math:gemini-decision-1_en",
    "math:gemini-decision-1_en_dontknow"
   ],
   "notes": "hosted API via OpenRouter",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "fastino-glide",
   "name": "GLiDE",
   "display_name": "GLiDE",
   "tag": "API",
   "class_label": "Closed · Efficient",
   "base_model": null,
   "setup_label": "Fastino GLiDE",
   "maker": "Fastino",
   "kind": "decision",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 63.47665279704115,
   "raw": {
    "metric": "accuracy",
    "value": 0.7078132223763292,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.05037002712663995,
   "improvement_points": null,
   "runs": [
    "math:glide_en",
    "math:glide_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": "vllm-sr/d3",
   "commit": "20f37b9a892acc13d4b07a4f3c4f11b2518c615c",
   "model_key": "vllm-sr-d3",
   "name": "d3",
   "display_name": "d3",
   "tag": "",
   "class_label": "Open · 26B",
   "base_model": null,
   "setup_label": "vLLM-SR d3",
   "maker": "vLLM Semantic Router",
   "kind": "decision",
   "params_total": 26.09,
   "params_active": 26.09,
   "quantization": "BF16",
   "score_0_100": 63.01433194637076,
   "raw": {
    "metric": "accuracy",
    "value": 0.7041146555709662,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.04492932511844977,
   "improvement_points": null,
   "runs": [
    "math:d3_en",
    "math:d3_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.035277777777777776,
   "gpu_cost_usd": 0.08784166666666667
  },
  {
   "repo": "nace-ai/drex-max-v1",
   "commit": "cac65e1982a0237e996419ba559004522d9a33ac",
   "model_key": "drex-max-v1",
   "name": "Drex Max",
   "display_name": "Drex Max",
   "tag": "",
   "class_label": "Open · 27B",
   "base_model": null,
   "setup_label": "Drex Max v1",
   "maker": "Nace.AI",
   "kind": "decision",
   "params_total": 27.0,
   "params_active": 27.0,
   "quantization": "BF16",
   "score_0_100": 62.898751733703186,
   "raw": {
    "metric": "accuracy",
    "value": 0.7031900138696255,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.11814199425449926,
   "improvement_points": null,
   "runs": [
    "math:drex-max-v1_en",
    "math:drex-max-v1_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); served with Nace's drex-decision-models code at e206b27",
   "gpu_hours": 0.0475,
   "gpu_cost_usd": 0.118275
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "drex-max-1.1",
   "name": "Drex Max v1.1",
   "display_name": "Drex Max v1.1",
   "tag": "",
   "class_label": "Closed · Efficient (provisional)",
   "base_model": null,
   "setup_label": "Drex Max 1.1",
   "maker": "Nace.AI",
   "kind": "decision",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 62.60980120203421,
   "raw": {
    "metric": "accuracy",
    "value": 0.7008784096162737,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.056972445603077154,
   "improvement_points": null,
   "runs": [
    "math:drex-max-1.1_en",
    "math:drex-max-1.1_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "celeris-1-decision",
   "name": "celeris-1-decision",
   "display_name": "celeris-1-decision",
   "tag": "",
   "class_label": "Closed · Efficient",
   "base_model": null,
   "setup_label": "Celeris-1 Decision",
   "maker": "Celeris",
   "kind": "decision",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 62.49422098936661,
   "raw": {
    "metric": "accuracy",
    "value": 0.6999537679149329,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.04310679611650484,
   "improvement_points": null,
   "runs": [
    "math:celeris_en",
    "math:celeris_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": "fastino/GLiDE",
   "commit": "a7e45896be7c96ba3354891da790c5ec99c30b6c",
   "model_key": "fastino-glide-v1.2",
   "name": "GLiDE",
   "display_name": "GLiDE",
   "tag": "v1.2",
   "class_label": "Open · 27B",
   "base_model": null,
   "setup_label": "Fastino GLiDE v1.2",
   "maker": "Fastino",
   "kind": "decision",
   "params_total": 27.0,
   "params_active": 27.0,
   "quantization": "BF16",
   "score_0_100": 62.320850670365225,
   "raw": {
    "metric": "accuracy",
    "value": 0.6985668053629218,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.05230666363357367,
   "improvement_points": null,
   "runs": [
    "math:glide-v1.2_en",
    "math:glide-v1.2_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.050277777777777775,
   "gpu_cost_usd": 0.12519166666666667
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "celeris-1.1-decision",
   "name": "celeris-1.1-decision",
   "display_name": "celeris-1.1-decision",
   "tag": "",
   "class_label": "Closed · Efficient",
   "base_model": null,
   "setup_label": "Celeris-1.1 Decision",
   "maker": "Celeris",
   "kind": "decision",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 62.320850670365225,
   "raw": {
    "metric": "accuracy",
    "value": 0.6985668053629218,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.04499768839574669,
   "improvement_points": null,
   "runs": [
    "math:celeris-1.1_en",
    "math:celeris-1.1_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "liquid-d1.2-ultra",
   "name": "d1.2-ultra",
   "display_name": "d1.2-ultra",
   "tag": "",
   "class_label": "Closed · Undisclosed",
   "base_model": null,
   "setup_label": "Liquid d1.2-ultra",
   "maker": "Liquid AI",
   "kind": "decision",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 62.320850670365225,
   "raw": {
    "metric": "accuracy",
    "value": 0.6985668053629218,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.04578252722217225,
   "improvement_points": null,
   "runs": [
    "math:d1.2-ultra_en",
    "math:d1.2-ultra_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "perplexity-decider-27b-v1.2",
   "name": "Perplexity Decider 27B v1.2",
   "display_name": "Perplexity Decider 27B v1.2",
   "tag": "API",
   "class_label": "Open · 27B",
   "base_model": null,
   "setup_label": "Perplexity Decider 27B v1.2 (API)",
   "maker": "Perplexity",
   "kind": "decision",
   "params_total": 27.0,
   "params_active": 27.0,
   "quantization": null,
   "score_0_100": 62.20527045769764,
   "raw": {
    "metric": "accuracy",
    "value": 0.6976421636615812,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.03229675297431705,
   "improvement_points": null,
   "runs": [
    "math:decider-v1.2_en",
    "math:decider-v1.2_en_dontknow"
   ],
   "notes": "hosted API (the maker's service); other setups of this model under its row, not in this entry: Perplexity Decider 27B v1.2 (NVIDIA RTX PRO 6000 Blackwell)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "perplexity-decider-27b-v1.2-with-web",
   "name": "Perplexity Decider 27B v1.2",
   "display_name": "Perplexity Decider 27B v1.2",
   "tag": "API · with web",
   "class_label": "Open · 27B",
   "base_model": null,
   "setup_label": "Perplexity Decider 27B v1.2 (API, with web)",
   "maker": "Perplexity",
   "kind": "decision",
   "params_total": 27.0,
   "params_active": 27.0,
   "quantization": null,
   "score_0_100": 61.627369394359675,
   "raw": {
    "metric": "accuracy",
    "value": 0.6930189551548774,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.03698135910311147,
   "improvement_points": null,
   "runs": [
    "math:decider-v1.2-web_en",
    "math:decider-v1.2-web_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": "turion-ai/turion-jev-31b",
   "commit": "866c0fc6e040cba23fcc72113a80e5c99d5ebab1",
   "model_key": "turion-jev-31b",
   "name": "Turion Jev 31B",
   "display_name": "Turion Jev 31B",
   "tag": "",
   "class_label": "Open · 31B",
   "base_model": null,
   "setup_label": "Turion Jev 31B",
   "maker": "Turion.AI",
   "kind": "decision",
   "params_total": 31.0,
   "params_active": 31.0,
   "quantization": "FP8",
   "score_0_100": 61.280628756356904,
   "raw": {
    "metric": "accuracy",
    "value": 0.6902450300508552,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.046954380490060094,
   "improvement_points": null,
   "runs": [
    "math:turion-jev-31b_en",
    "math:turion-jev-31b_en_dontknow"
   ],
   "notes": "on our NVIDIA DGX Spark (GB10)",
   "gpu_hours": 0.1775,
   "gpu_cost_usd": 0.037775164130024005
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "subq-snap-1.1",
   "name": "SubQ Snap 1.1",
   "display_name": "SubQ Snap 1.1",
   "tag": "",
   "class_label": "Closed · Efficient",
   "base_model": null,
   "setup_label": "SubQ Snap 1.1",
   "maker": "Subquadratic",
   "kind": "decision",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 60.70272769301896,
   "raw": {
    "metric": "accuracy",
    "value": 0.6856218215441516,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.095208335208,
   "improvement_points": null,
   "runs": [
    "math:subq-1.1_en",
    "math:subq-1.1_en_dontknow"
   ],
   "notes": "hosted API (the maker's service); shown as one row with SubQ Snap Preview (the same answer to every question); every number from SubQ Snap 1.1",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": "PelaAI/KnowLine-12B-Gen1",
   "commit": "f26fbc2d64092bb2f8caaedbec243172bb3e908f",
   "model_key": "knowline-12b-gen1",
   "name": "KnowLine-12B-Gen1",
   "display_name": "KnowLine-12B-Gen1",
   "tag": "",
   "class_label": "Open · 12B",
   "base_model": "Gemma 4 12B",
   "setup_label": "KnowLine-12B-Gen1",
   "maker": "PelaAI",
   "kind": "decision",
   "params_total": 12.0,
   "params_active": 12.0,
   "quantization": "BF16",
   "score_0_100": 57.813222376329165,
   "raw": {
    "metric": "accuracy",
    "value": 0.6625057790106333,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.0397398558622103,
   "improvement_points": null,
   "runs": [
    "math:knowline-12b-gen1_en",
    "math:knowline-12b-gen1_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.03138888888888889,
   "gpu_cost_usd": 0.07815833333333333
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "liquid-d1.2",
   "name": "d1.2",
   "display_name": "d1.2",
   "tag": "",
   "class_label": "Closed · Undisclosed",
   "base_model": null,
   "setup_label": "Liquid d1.2",
   "maker": "Liquid AI",
   "kind": "decision",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 56.88858067498844,
   "raw": {
    "metric": "accuracy",
    "value": 0.6551086453999075,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.0322345924099891,
   "improvement_points": null,
   "runs": [
    "math:d1.2_en",
    "math:d1.2_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "liquid-d1.1",
   "name": "d1.1",
   "display_name": "d1.1",
   "tag": "",
   "class_label": "Closed · Efficient",
   "base_model": null,
   "setup_label": "Liquid d1.1",
   "maker": "Liquid AI",
   "kind": "decision",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 56.48404993065187,
   "raw": {
    "metric": "accuracy",
    "value": 0.651872399445215,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.021643242997207185,
   "improvement_points": null,
   "runs": [
    "math:d1.1_en",
    "math:d1.1_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "jev-1.14.0",
   "name": "Jev 1.14",
   "display_name": "Jev 1.14",
   "tag": "",
   "class_label": "Closed · Efficient",
   "base_model": null,
   "setup_label": "Jev 1.14.0",
   "maker": "TypeSafe",
   "kind": "decision",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 56.42625982431807,
   "raw": {
    "metric": "accuracy",
    "value": 0.6514100785945446,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.028169209431345345,
   "improvement_points": null,
   "runs": [
    "math:jev-1.14_en",
    "math:jev-1.14_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "jev-1.13.0",
   "name": "Jev 1.13",
   "display_name": "Jev 1.13",
   "tag": "",
   "class_label": "Closed · Efficient",
   "base_model": null,
   "setup_label": "Jev 1.13.0",
   "maker": "TypeSafe",
   "kind": "decision",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 55.09708737864077,
   "raw": {
    "metric": "accuracy",
    "value": 0.6407766990291263,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.025681923254738812,
   "improvement_points": null,
   "runs": [
    "math:jev-1.13_en",
    "math:jev-1.13_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": "newsbang/adjudex-9b-v5",
   "commit": "ffdcb8254d1b22e40b2bbbf569f143e0b357aeca",
   "model_key": "adjudex-9b-v5",
   "name": "Adjudex 9B v5",
   "display_name": "Adjudex 9B v5",
   "tag": "",
   "class_label": "Open · 9.0B",
   "base_model": "Qwen3.5-9B",
   "setup_label": "Adjudex 9B v5",
   "maker": "newsbang",
   "kind": "decision",
   "params_total": 9.0,
   "params_active": 9.0,
   "quantization": "BF16",
   "score_0_100": 52.612112806287556,
   "raw": {
    "metric": "accuracy",
    "value": 0.6208969024503005,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.04793390671264879,
   "improvement_points": null,
   "runs": [
    "math:adjudex-9b_en",
    "math:adjudex-9b_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.023055555555555555,
   "gpu_cost_usd": 0.05740833333333334
  },
  {
   "repo": "baselabs/elephant-1",
   "commit": "b3f8a93462fb1051a9de031e94a7ff380e254072",
   "model_key": "elephant-1",
   "name": "Elephant-1",
   "display_name": "Elephant-1",
   "tag": "",
   "class_label": "Open · 9.0B",
   "base_model": null,
   "setup_label": "Elephant-1",
   "maker": "Base Labs",
   "kind": "decision",
   "params_total": 9.0,
   "params_active": 9.0,
   "quantization": "BF16",
   "score_0_100": 51.68747110494684,
   "raw": {
    "metric": "accuracy",
    "value": 0.6134997688395747,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.042894290799815056,
   "improvement_points": null,
   "runs": [
    "math:elephant-1_en",
    "math:elephant-1_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.050555555555555555,
   "gpu_cost_usd": 0.12588333333333335
  },
  {
   "repo": "PelaAI/KnowLine-9B-Gen2",
   "commit": "8ec8eb609bc10aa6050ae3754aee419d458fae42",
   "model_key": "knowline-9b-gen2",
   "name": "KnowLine-9B-Gen2",
   "display_name": "KnowLine-9B-Gen2",
   "tag": "",
   "class_label": "Open · 9.0B",
   "base_model": "Qwen3.5-9B",
   "setup_label": "KnowLine-9B-Gen2",
   "maker": "PelaAI",
   "kind": "decision",
   "params_total": 9.0,
   "params_active": 9.0,
   "quantization": "BF16",
   "score_0_100": 50.12713823393435,
   "raw": {
    "metric": "accuracy",
    "value": 0.6010171058714748,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.047920111962225155,
   "improvement_points": null,
   "runs": [
    "math:knowline-9b-gen2_en",
    "math:knowline-9b-gen2_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.026944444444444444,
   "gpu_cost_usd": 0.06709166666666667
  },
  {
   "repo": "tattrongvu/Jeffan-9B-1211",
   "commit": "862d1a7a8b64a8d708ddb0b0cd775a740f01841a",
   "model_key": "jeffan-9b-1211",
   "name": "Jeffan-9B-1211",
   "display_name": "Jeffan-9B-1211",
   "tag": "",
   "class_label": "Open · 9.0B",
   "base_model": "Qwen3.5-9B",
   "setup_label": "Jeffan-9B-1211",
   "maker": "tattrongvu",
   "kind": "decision",
   "params_total": 9.0,
   "params_active": 9.0,
   "quantization": "BF16",
   "score_0_100": 49.780397595931575,
   "raw": {
    "metric": "accuracy",
    "value": 0.5982431807674526,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.037523772652044284,
   "improvement_points": null,
   "runs": [
    "math:jeffan-9b_en",
    "math:jeffan-9b_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.025,
   "gpu_cost_usd": 0.06225000000000001
  },
  {
   "repo": "PelaAI/KnowLine-9B-Gen1",
   "commit": "325378d128e9f5eb5088196668f5874c5682b6c0",
   "model_key": "knowline-9b-gen1",
   "name": "KnowLine-9B-Gen1",
   "display_name": "KnowLine-9B-Gen1",
   "tag": "",
   "class_label": "Open · 9.0B",
   "base_model": "Qwen3.5-9B",
   "setup_label": "KnowLine-9B-Gen1",
   "maker": "PelaAI",
   "kind": "decision",
   "params_total": 9.0,
   "params_active": 9.0,
   "quantization": "BF16",
   "score_0_100": 49.72260748959777,
   "raw": {
    "metric": "accuracy",
    "value": 0.5977808599167822,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.030535655418292937,
   "improvement_points": null,
   "runs": [
    "math:knowline-9b_en",
    "math:knowline-9b_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.023055555555555555,
   "gpu_cost_usd": 0.05740833333333334
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "microsoft-decision-1",
   "name": "Microsoft-Decision-1",
   "display_name": "Microsoft-Decision-1",
   "tag": "",
   "class_label": "Closed · Efficient",
   "base_model": "Qwen3.5-9B",
   "setup_label": "Microsoft-Decision-1",
   "maker": "Microsoft",
   "kind": "decision",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 49.20249653259361,
   "raw": {
    "metric": "accuracy",
    "value": 0.5936199722607489,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.040862520669752544,
   "improvement_points": null,
   "runs": [
    "math:ms-decision-1_en",
    "math:ms-decision-1_en_dontknow"
   ],
   "notes": "hosted API via OpenRouter",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "drex-2.0",
   "name": "Drex 2.0",
   "display_name": "Drex 2.0",
   "tag": "",
   "class_label": "Open · 35B MoE · 3.0B active",
   "base_model": null,
   "setup_label": "Drex 2.0",
   "maker": "Nace.AI",
   "kind": "decision",
   "params_total": 35.0,
   "params_active": 3.0,
   "quantization": null,
   "score_0_100": 48.740175681923255,
   "raw": {
    "metric": "accuracy",
    "value": 0.5899214054553861,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.01879144706426259,
   "improvement_points": null,
   "runs": [
    "math:drex-2.0_en",
    "math:drex-2.0_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": "Patrick-18/Dexy-Decision-9B-V2",
   "commit": "4f434e2b5fd9f0addfe3233489571c2dc8ce7a19",
   "model_key": "dexy-decision-9b-v2",
   "name": "Dexy-Decision-9B-V2",
   "display_name": "Dexy-Decision-9B-V2",
   "tag": "",
   "class_label": "Open · 9.0B",
   "base_model": "vllm-sr/d3-flash",
   "setup_label": "Dexy-Decision-9B-V2",
   "maker": "Patrick-18",
   "kind": "decision",
   "params_total": 9.0,
   "params_active": 9.0,
   "quantization": "BF16",
   "score_0_100": 48.509015256588064,
   "raw": {
    "metric": "accuracy",
    "value": 0.5880721220527045,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.12131842805076583,
   "improvement_points": null,
   "runs": [
    "math:dexy-9b_en",
    "math:dexy-9b_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.027777777777777776,
   "gpu_cost_usd": 0.06916666666666667
  },
  {
   "repo": "startlux-models/StartLux-Decision-v1.1-9B-Instant",
   "commit": "18a17f734966b213b2f00fa9c915e8b9af2a9b20",
   "model_key": "startlux-decision-v1.1-9b",
   "name": "StartLux-Decision-v1.1-9B-Instant",
   "display_name": "StartLux-Decision-v1.1-9B-Instant",
   "tag": "",
   "class_label": "Open · 9.0B",
   "base_model": null,
   "setup_label": "StartLux-Decision v1.1 9B",
   "maker": "StartLux",
   "kind": "decision",
   "params_total": 9.0,
   "params_active": 9.0,
   "quantization": "BF16",
   "score_0_100": 48.046694405917705,
   "raw": {
    "metric": "accuracy",
    "value": 0.5843735552473417,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.02693892628266125,
   "improvement_points": null,
   "runs": [
    "math:startlux-9b_en",
    "math:startlux-9b_en_dontknow"
   ],
   "notes": "on our NVIDIA DGX Spark (GB10)",
   "gpu_hours": 0.08472222222222223,
   "gpu_cost_usd": 0.018030399154393303
  },
  {
   "repo": "fastino/GLiDE-9B",
   "commit": "ab54df5acba0eebf7df45560ffdaacd64e32df3b",
   "model_key": "glide-9b",
   "name": "GLiDE 9B",
   "display_name": "GLiDE 9B",
   "tag": "",
   "class_label": "Open · 7.9B",
   "base_model": null,
   "setup_label": "GLiDE 9B (open)",
   "maker": "Fastino",
   "kind": "decision",
   "params_total": 7.94,
   "params_active": 7.94,
   "quantization": "BF16",
   "score_0_100": 47.41100323624595,
   "raw": {
    "metric": "accuracy",
    "value": 0.5792880258899676,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.070737570513874,
   "improvement_points": null,
   "runs": [
    "math:glide-9b_en",
    "math:glide-9b_en_dontknow"
   ],
   "notes": "on our NVIDIA DGX Spark (GB10)",
   "gpu_hours": 0.10666666666666667,
   "gpu_cost_usd": 0.022700568115695176
  },
  {
   "repo": "scar-ai/vera-medium-omni",
   "commit": "1f23a6cf5304e8e53b7ded53980765d5f8f4047c",
   "model_key": "vera-medium-omni",
   "name": "vera-medium-omni",
   "display_name": "vera-medium-omni",
   "tag": "",
   "class_label": "Open · 4.2B",
   "base_model": "Qwen3.5-4B",
   "setup_label": "vera-medium-omni",
   "maker": "scar-ai",
   "kind": "decision",
   "params_total": 4.2,
   "params_active": 4.2,
   "quantization": "BF16",
   "score_0_100": 46.024040684234855,
   "raw": {
    "metric": "accuracy",
    "value": 0.5681923254738789,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.029700739713361058,
   "improvement_points": null,
   "runs": [
    "math:vera-medium-omni_en",
    "math:vera-medium-omni_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.020277777777777777,
   "gpu_cost_usd": 0.05049166666666667
  },
  {
   "repo": "apertixai/cerno-9b-v1.2",
   "commit": "f02a632fbe3a4698cd1ff5c0b929ea240e2b8eda",
   "model_key": "cerno-9b-v1.2",
   "name": "Cerno 9B v1.2",
   "display_name": "Cerno 9B v1.2",
   "tag": "",
   "class_label": "Open · 9.0B",
   "base_model": null,
   "setup_label": "Cerno 9B v1.2",
   "maker": "Apertix AI",
   "kind": "decision",
   "params_total": 9.0,
   "params_active": 9.0,
   "quantization": "BF16",
   "score_0_100": 45.041608876560325,
   "raw": {
    "metric": "accuracy",
    "value": 0.5603328710124826,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.15733795654184,
   "improvement_points": null,
   "runs": [
    "math:cerno-9b_en",
    "math:cerno-9b_en_dontknow"
   ],
   "notes": "on our NVIDIA DGX Spark (GB10)",
   "gpu_hours": 0.09666666666666666,
   "gpu_cost_usd": 0.020572389854848754
  },
  {
   "repo": "PelaAI/KnowLine-4B-Gen5.5",
   "commit": "779944d2cc8ffa19cef386aee83c83bcb967ade2",
   "model_key": "knowline-4b-gen5.5",
   "name": "KnowLine 4B Gen5.5",
   "display_name": "KnowLine 4B Gen5.5",
   "tag": "",
   "class_label": "Open · 4.0B",
   "base_model": null,
   "setup_label": "KnowLine 4B Gen5.5",
   "maker": "PelaAI",
   "kind": "decision",
   "params_total": 4.0,
   "params_active": 4.0,
   "quantization": "FP8",
   "score_0_100": 44.983818770226534,
   "raw": {
    "metric": "accuracy",
    "value": 0.5598705501618123,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.08632431691710336,
   "improvement_points": null,
   "runs": [
    "math:knowline-4b_en",
    "math:knowline-4b_en_dontknow"
   ],
   "notes": "on our NVIDIA DGX Spark (GB10)",
   "gpu_hours": 0.04638888888888889,
   "gpu_cost_usd": 0.00987238248781535
  },
  {
   "repo": "perplexity-ai/pplx-decider-flash-v1.3",
   "commit": "d06fb3f2d97e0a52325a0d463c450cf0338f44fe",
   "model_key": "perplexity-decider-flash-v1.3",
   "name": "Perplexity Decider Flash v1.3",
   "display_name": "Perplexity Decider Flash v1.3",
   "tag": "",
   "class_label": "Open · 4.0B",
   "base_model": null,
   "setup_label": "Perplexity Decider Flash v1.3",
   "maker": "Perplexity",
   "kind": "decision",
   "params_total": 4.0,
   "params_active": 4.0,
   "quantization": "BF16",
   "score_0_100": 44.926028663892744,
   "raw": {
    "metric": "accuracy",
    "value": 0.559408229311142,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.1167804804730328,
   "improvement_points": null,
   "runs": [
    "math:decider-flash-v1.3_en",
    "math:decider-flash-v1.3_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.019444444444444445,
   "gpu_cost_usd": 0.04841666666666667
  },
  {
   "repo": "startlux-models/StartLux-Decision-v1.1-4B-Instant",
   "commit": "416088f77cfce09633906a3675c7cd2e5dabcf10",
   "model_key": "startlux-decision-v1.1-4b",
   "name": "StartLux-Decision-v1.1-4B-Instant",
   "display_name": "StartLux-Decision-v1.1-4B-Instant",
   "tag": "",
   "class_label": "Open · 4.0B",
   "base_model": null,
   "setup_label": "StartLux-Decision v1.1 4B",
   "maker": "StartLux",
   "kind": "decision",
   "params_total": 4.0,
   "params_active": 4.0,
   "quantization": "BF16",
   "score_0_100": 44.81044845122515,
   "raw": {
    "metric": "accuracy",
    "value": 0.5584835876098012,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.06650363539344888,
   "improvement_points": null,
   "runs": [
    "math:startlux-4b_en",
    "math:startlux-4b_en_dontknow"
   ],
   "notes": "on our NVIDIA DGX Spark (GB10)",
   "gpu_hours": 0.041666666666666664,
   "gpu_cost_usd": 0.008867409420193428
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "liquid-d1",
   "name": "d1",
   "display_name": "d1",
   "tag": "",
   "class_label": "Closed · Efficient",
   "base_model": null,
   "setup_label": "Liquid d1",
   "maker": "Liquid AI",
   "kind": "decision",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 44.52149791955616,
   "raw": {
    "metric": "accuracy",
    "value": 0.5561719833564494,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.032904607617458656,
   "improvement_points": null,
   "runs": [
    "math:d1_en",
    "math:d1_en_dontknow"
   ],
   "notes": "hosted API (the maker's service)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": "Valen-Team/Valen-Decision-1.5-9B",
   "commit": "5a512117ebc0f2b69f311becfcf47cdef8abf568",
   "model_key": "valen-decision-1.5-9b",
   "name": "Valen-Decision-1.5-9B",
   "display_name": "Valen-Decision-1.5-9B",
   "tag": "",
   "class_label": "Open · 9.0B",
   "base_model": "Qwen3.5-9B",
   "setup_label": "Valen-Decision-1.5-9B",
   "maker": "Valen-Team",
   "kind": "decision",
   "params_total": 9.0,
   "params_active": 9.0,
   "quantization": "BF16",
   "score_0_100": 44.17475728155339,
   "raw": {
    "metric": "accuracy",
    "value": 0.5533980582524272,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.040623399081564475,
   "improvement_points": null,
   "runs": [
    "math:valen-9b_en",
    "math:valen-9b_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.035555555555555556,
   "gpu_cost_usd": 0.08853333333333334
  },
  {
   "repo": "perplexity-ai/pplx-decider-flash-v1.2",
   "commit": "6d771ad7b5ecff5e93b8fc7b90d4c5ccc341954d",
   "model_key": "perplexity-decider-flash-v1.2",
   "name": "Perplexity Decider Flash v1.2",
   "display_name": "Perplexity Decider Flash v1.2",
   "tag": "",
   "class_label": "Open · 4.0B",
   "base_model": null,
   "setup_label": "Perplexity Decider Flash v1.2",
   "maker": "Perplexity",
   "kind": "decision",
   "params_total": 4.0,
   "params_active": 4.0,
   "quantization": "BF16",
   "score_0_100": 44.05917706888581,
   "raw": {
    "metric": "accuracy",
    "value": 0.5524734165510865,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.10375862150665573,
   "improvement_points": null,
   "runs": [
    "math:decider-flash-v1.2_en",
    "math:decider-flash-v1.2_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.026111111111111113,
   "gpu_cost_usd": 0.06501666666666668
  },
  {
   "repo": "ZefanCai/Open-Jev-27B-v1.1",
   "commit": "28cf73067d5b337860bbef3c85b8b82ba8730956",
   "model_key": "open-jev-27b-v1.1",
   "name": "Open-Jev-27B-v1.1",
   "display_name": "Open-Jev-27B-v1.1",
   "tag": "",
   "class_label": "Open · 27B",
   "base_model": "Qwen3.8-27B + adapter",
   "setup_label": "Open-Jev 27B v1.1",
   "maker": "Zefan Cai",
   "kind": "decision",
   "params_total": 27.0,
   "params_active": 27.0,
   "quantization": "BF16",
   "score_0_100": 43.192325473878874,
   "raw": {
    "metric": "accuracy",
    "value": 0.545538603791031,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.21052697404071077,
   "improvement_points": null,
   "runs": [
    "math:open-jev_en",
    "math:open-jev_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); the repository holds an adapter and head on Qwen3.8-27B",
   "gpu_hours": 0.14694444444444443,
   "gpu_cost_usd": 0.36589166666666667
  },
  {
   "repo": "OmniJev/OneJev-27B",
   "commit": "a5e5f55bdbdc9deda44f3fcb5b163776b9b369e8",
   "model_key": "onejev-27b",
   "name": "OneJev-27B",
   "display_name": "OneJev-27B",
   "tag": "",
   "class_label": "Open · 27B",
   "base_model": null,
   "setup_label": "OneJev 27B",
   "maker": "OmniJev",
   "kind": "decision",
   "params_total": 27.0,
   "params_active": 27.0,
   "quantization": "BF16",
   "score_0_100": 42.209893666204344,
   "raw": {
    "metric": "accuracy",
    "value": 0.5376791493296348,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.03582799722607488,
   "improvement_points": null,
   "runs": [
    "math:onejev_en",
    "math:onejev_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.05277777777777778,
   "gpu_cost_usd": 0.13141666666666668
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "inception-mercury-decide",
   "name": "Mercury Decide",
   "display_name": "Mercury Decide",
   "tag": "",
   "class_label": "Closed · Efficient",
   "base_model": null,
   "setup_label": "Inception Mercury Decide",
   "maker": "Inception",
   "kind": "decision",
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 41.86315302820157,
   "raw": {
    "metric": "accuracy",
    "value": 0.5349052242256126,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.15801202201424802,
   "improvement_points": null,
   "runs": [
    "math:mercury-decide_en",
    "math:mercury-decide_en_dontknow"
   ],
   "notes": "hosted API via OpenRouter",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "repo": "victorbennett/Verdin-4B-v1.1",
   "commit": "45830d216bb7e45f43f0a482c131a7cafa6d189f",
   "model_key": "verdin-4b-v1.1",
   "name": "Verdin-4B-v1.1",
   "display_name": "Verdin-4B-v1.1",
   "tag": "",
   "class_label": "Open · 4.0B",
   "base_model": "Qwen3.5-4B",
   "setup_label": "Verdin-4B-v1.1",
   "maker": "victorbennett",
   "kind": "decision",
   "params_total": 4.0,
   "params_active": 4.0,
   "quantization": "BF16",
   "score_0_100": 39.37817845584835,
   "raw": {
    "metric": "accuracy",
    "value": 0.5150254276467868,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.056776592633318794,
   "improvement_points": null,
   "runs": [
    "math:verdin-4b_en",
    "math:verdin-4b_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.029166666666666667,
   "gpu_cost_usd": 0.07262500000000001
  },
  {
   "repo": "Cloudflare/clef",
   "commit": "2f3de3dd85f379784083b0814d997ab627200f0c",
   "model_key": "clef",
   "name": "Clef",
   "display_name": "Clef",
   "tag": "",
   "class_label": "Open · 27B",
   "base_model": null,
   "setup_label": "Clef",
   "maker": "Cloudflare",
   "kind": "decision",
   "params_total": 27.0,
   "params_active": 27.0,
   "quantization": "BF16",
   "score_0_100": 38.85806749884419,
   "raw": {
    "metric": "accuracy",
    "value": 0.5108645399907535,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.10962413314840498,
   "improvement_points": null,
   "runs": [
    "math:clef_en",
    "math:clef_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.11416666666666667,
   "gpu_cost_usd": 0.28427500000000006
  },
  {
   "repo": "quincyqiang/chinese-decision-pointer-4b",
   "commit": "889f23e2655e5cba1208bb17f90526092813f5f2",
   "model_key": "chinese-decision-pointer-4b",
   "name": "Chinese Decision Pointer 4B",
   "display_name": "Chinese Decision Pointer 4B",
   "tag": "",
   "class_label": "Open · 4.0B",
   "base_model": "Qwen3.5-4B",
   "setup_label": "Chinese Decision Pointer 4B",
   "maker": "quincyqiang",
   "kind": "decision",
   "params_total": 4.0,
   "params_active": 4.0,
   "quantization": "BF16",
   "score_0_100": 38.04900601017106,
   "raw": {
    "metric": "accuracy",
    "value": 0.5043920480813685,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.1374209486331622,
   "improvement_points": null,
   "runs": [
    "math:chinese-pointer-4b_en",
    "math:chinese-pointer-4b_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.01972222222222222,
   "gpu_cost_usd": 0.04910833333333334
  },
  {
   "repo": "Cloudflare/clef-flash",
   "commit": "fde727a287004204b7518dcc983fe64379776712",
   "model_key": "clef-flash",
   "name": "Clef-Flash",
   "display_name": "Clef-Flash",
   "tag": "",
   "class_label": "Open · 9.0B",
   "base_model": null,
   "setup_label": "Clef-flash",
   "maker": "Cloudflare",
   "kind": "decision",
   "params_total": 9.0,
   "params_active": 9.0,
   "quantization": "BF16",
   "score_0_100": 37.23994452149791,
   "raw": {
    "metric": "accuracy",
    "value": 0.49791955617198336,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.026981692094313482,
   "improvement_points": null,
   "runs": [
    "math:clef-flash_en",
    "math:clef-flash_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.0625,
   "gpu_cost_usd": 0.155625
  },
  {
   "repo": "EldanRing/Winnow-12B",
   "commit": "b2b14213dfa252e6d6b543c8b334762e51772d29",
   "model_key": "winnow-12b",
   "name": "Winnow-12B",
   "display_name": "Winnow-12B",
   "tag": "",
   "class_label": "Open · 12B",
   "base_model": null,
   "setup_label": "Winnow-12B",
   "maker": "EldanRing",
   "kind": "decision",
   "params_total": 12.0,
   "params_active": 12.0,
   "quantization": "Q8_0",
   "score_0_100": 33.599167822468786,
   "raw": {
    "metric": "accuracy",
    "value": 0.46879334257975036,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.0839036235732786,
   "improvement_points": null,
   "runs": [
    "math:winnow_en",
    "math:winnow_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); the Q8_0 GGUF, served by winnow-inference at d4631fb",
   "gpu_hours": 0.03333333333333333,
   "gpu_cost_usd": 0.083
  },
  {
   "repo": null,
   "commit": null,
   "model_key": "cygnet",
   "name": "Cygnet",
   "display_name": "Cygnet",
   "tag": "",
   "class_label": "Open · 12B",
   "base_model": "Gemma 4 12B IT",
   "setup_label": "Cygnet",
   "maker": "Blockbrain Labs",
   "kind": "decision",
   "params_total": 11.95,
   "params_active": 11.95,
   "quantization": "BF16",
   "score_0_100": 30.42071197411003,
   "raw": {
    "metric": "accuracy",
    "value": 0.44336569579288027,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.059426522048276566,
   "improvement_points": null,
   "runs": [
    "math:cygnet_en",
    "math:cygnet_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); no weights of its own: google/gemma-4-12B-it at 707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7, unchanged, behind cygnet-recipe's shim (github.com/blockbrain-ai/cygnet-recipe 3cf591c, temperature 3.4); repo left empty so it is not matched with Gemma 4 12B",
   "gpu_hours": 0.02388888888888889,
   "gpu_cost_usd": 0.05948333333333334
  },
  {
   "repo": "wayfind/metask-jev-4b-policy-mix",
   "commit": "ea20fe85b28733b1522721dec119bec50947a869",
   "model_key": "metask-jev-4b",
   "name": "Metask-Jev-4B",
   "display_name": "Metask-Jev-4B",
   "tag": "",
   "class_label": "Open · 4.5B",
   "base_model": null,
   "setup_label": "Metask-Jev 4B",
   "maker": "Metask Lab",
   "kind": "decision",
   "params_total": 4.54,
   "params_active": 4.54,
   "quantization": "BF16",
   "score_0_100": 23.948220064724918,
   "raw": {
    "metric": "accuracy",
    "value": 0.39158576051779936,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.03300643423283703,
   "improvement_points": null,
   "runs": [
    "math:metask_en",
    "math:metask_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.07222222222222222,
   "gpu_cost_usd": 0.17983333333333335
  },
  {
   "repo": "startlux-models/StartLux-Decision-v1.1-2B-Instant",
   "commit": "f1e6cd147dda82c9ffa3e8e725d5c1f175566fb1",
   "model_key": "startlux-decision-v1.1-2b",
   "name": "StartLux-Decision-v1.1-2B-Instant",
   "display_name": "StartLux-Decision-v1.1-2B-Instant",
   "tag": "",
   "class_label": "Open · 2.0B",
   "base_model": null,
   "setup_label": "StartLux-Decision v1.1 2B",
   "maker": "StartLux",
   "kind": "decision",
   "params_total": 2.0,
   "params_active": 2.0,
   "quantization": "BF16",
   "score_0_100": 23.312528895053163,
   "raw": {
    "metric": "accuracy",
    "value": 0.3865002311604253,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.026080989749775452,
   "improvement_points": null,
   "runs": [
    "math:startlux-2b_en",
    "math:startlux-2b_en_dontknow"
   ],
   "notes": "on our NVIDIA DGX Spark (GB10)",
   "gpu_hours": 0.023055555555555555,
   "gpu_cost_usd": 0.00490663321250703
  },
  {
   "repo": "LiquidAI/d1-2B",
   "commit": "10e7671d44e4b37700020b1eed3f594c5f2af0b4",
   "model_key": "liquid-d1-2b",
   "name": "d1-2B",
   "display_name": "d1-2B",
   "tag": "",
   "class_label": "Open · 1.9B",
   "base_model": "LFM3-VL-2B",
   "setup_label": "Liquid d1-2B (open weights)",
   "maker": "Liquid AI",
   "kind": "decision",
   "params_total": 1.92,
   "params_active": 1.92,
   "quantization": "BF16",
   "score_0_100": 21.347665279704113,
   "raw": {
    "metric": "accuracy",
    "value": 0.37078132223763294,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.05560965380005984,
   "improvement_points": null,
   "runs": [
    "math:d1-2b_en",
    "math:d1-2b_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB)",
   "gpu_hours": 0.012222222222222223,
   "gpu_cost_usd": 0.030433333333333333
  },
  {
   "repo": "fastino/GLiDE-2B",
   "commit": "00f122c120cdeed82284106cb77d9bab5a50ad16",
   "model_key": "glide-2b",
   "name": "GLiDE 2B",
   "display_name": "GLiDE 2B",
   "tag": "",
   "class_label": "Open · 1.9B",
   "base_model": null,
   "setup_label": "GLiDE 2B (open)",
   "maker": "Fastino",
   "kind": "decision",
   "params_total": 1.88,
   "params_active": 1.88,
   "quantization": "BF16",
   "score_0_100": 20.943134535367545,
   "raw": {
    "metric": "accuracy",
    "value": 0.36754507628294036,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.17155643235372448,
   "improvement_points": null,
   "runs": [
    "math:glide-2b_en",
    "math:glide-2b_en_dontknow"
   ],
   "notes": "on our NVIDIA DGX Spark (GB10)",
   "gpu_hours": 0.018055555555555554,
   "gpu_cost_usd": 0.0038425440820838187
  },
  {
   "repo": "PelaAI/KnowLine-2B-Gen1",
   "commit": "53245d470a4ce316e32fd0d18c24e142c697f40b",
   "model_key": "knowline-2b-gen1",
   "name": "KnowLine-2B-Gen1",
   "display_name": "KnowLine-2B-Gen1",
   "tag": "",
   "class_label": "Open · 2.0B",
   "base_model": "Qwen3.5-2B",
   "setup_label": "KnowLine-2B-Gen1",
   "maker": "PelaAI",
   "kind": "decision",
   "params_total": 2.0,
   "params_active": 2.0,
   "quantization": "BF16",
   "score_0_100": 19.09385113268608,
   "raw": {
    "metric": "accuracy",
    "value": 0.35275080906148865,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.09408551986577202,
   "improvement_points": null,
   "runs": [
    "math:knowline-2b_en",
    "math:knowline-2b_en_dontknow"
   ],
   "notes": "on our NVIDIA DGX Spark (GB10)",
   "gpu_hours": 0.0325,
   "gpu_cost_usd": 0.006916579347750875
  },
  {
   "repo": "fastino/GLiDE-0.8B",
   "commit": "9eacdeec79fa48ec52a8d6f54b331376b4bbfaf0",
   "model_key": "glide-0.8b",
   "name": "GLiDE 0.8B",
   "display_name": "GLiDE 0.8B",
   "tag": "",
   "class_label": "Open · 0.8B",
   "base_model": null,
   "setup_label": "GLiDE 0.8B (open)",
   "maker": "Fastino",
   "kind": "decision",
   "params_total": 0.75,
   "params_active": 0.75,
   "quantization": "BF16",
   "score_0_100": 15.10633379565418,
   "raw": {
    "metric": "accuracy",
    "value": 0.32085067036523346,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.18775447761695988,
   "improvement_points": null,
   "runs": [
    "math:glide-0.8b_en",
    "math:glide-0.8b_en_dontknow"
   ],
   "notes": "on our NVIDIA DGX Spark (GB10)",
   "gpu_hours": 0.013055555555555556,
   "gpu_cost_usd": 0.0027784549516606076
  },
  {
   "repo": "startlux-models/StartLux-Decision-v1.1-0.8B-Instant",
   "commit": "6496ae641cc71b149ce9dad69bd97f2a44e7bf11",
   "model_key": "startlux-decision-v1.1-0.8b-instant",
   "name": "StartLux-Decision-v1.1-0.8B-Instant",
   "display_name": "StartLux-Decision-v1.1-0.8B-Instant",
   "tag": "",
   "class_label": "Open · 0.8B",
   "base_model": "Qwen3.5-0.8B",
   "setup_label": "StartLux-Decision-v1.1-0.8B-Instant",
   "maker": "StartLux",
   "kind": "decision",
   "params_total": 0.8,
   "params_active": 0.8,
   "quantization": "BF16",
   "score_0_100": 12.101248266296807,
   "raw": {
    "metric": "accuracy",
    "value": 0.29680998613037446,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.04765301858447206,
   "improvement_points": null,
   "runs": [
    "math:startlux-0.8b_en",
    "math:startlux-0.8b_en_dontknow"
   ],
   "notes": "on our NVIDIA DGX Spark (GB10)",
   "gpu_hours": 0.015555555555555555,
   "gpu_cost_usd": 0.003310499516872213
  },
  {
   "repo": "StrandsAgents/strands-decider-2B-hobson-v21",
   "commit": "2b52a6235c1b8306bbfa30b00b9d4b74b63a39f5",
   "model_key": "strands-decider-2b",
   "name": "Strands Decider 2B",
   "display_name": "Strands Decider 2B",
   "tag": "",
   "class_label": "Open · 2.0B",
   "base_model": null,
   "setup_label": "Strands Decider 2B",
   "maker": "Strands Agents",
   "kind": "decision",
   "params_total": 2.0,
   "params_active": 2.0,
   "quantization": "BF16",
   "score_0_100": 10.829865926953307,
   "raw": {
    "metric": "accuracy",
    "value": 0.28663892741562647,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.0876370319001387,
   "improvement_points": null,
   "runs": [
    "math:strands_en",
    "math:strands_en_dontknow"
   ],
   "notes": "on our NVIDIA RTX PRO 6000 Blackwell (96 GB); the repository holds a LoRA and head on Qwen3.5-2B-Base",
   "gpu_hours": 0.02277777777777778,
   "gpu_cost_usd": 0.056716666666666665
  },
  {
   "repo": "OmniJev/OneJev1.6-0.8B",
   "commit": "bcd88a382f314b26cb7b1b7f369c1ed3502ee7a0",
   "model_key": "onejev1.6-0.8b",
   "name": "OneJev1.6-0.8B",
   "display_name": "OneJev1.6-0.8B",
   "tag": "",
   "class_label": "Open · 0.8B",
   "base_model": null,
   "setup_label": "OneJev1.6-0.8B",
   "maker": "OmniJev",
   "kind": "decision",
   "params_total": 0.8,
   "params_active": 0.8,
   "quantization": "BF16",
   "score_0_100": 10.309754969949147,
   "raw": {
    "metric": "accuracy",
    "value": 0.2824780397595932,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.09989208737864079,
   "improvement_points": null,
   "runs": [
    "math:onejev1.6-0.8b_en",
    "math:onejev1.6-0.8b_en_dontknow"
   ],
   "notes": "on our NVIDIA DGX Spark (GB10)",
   "gpu_hours": 0.011111111111111112,
   "gpu_cost_usd": 0.002364642512051581
  },
  {
   "repo": "PelaAI/KnowLine-0.8B-Gen1",
   "commit": "6b27831f7b783c03278bd946d4d8f4f7a98a3798",
   "model_key": "knowline-0.8b-gen1",
   "name": "KnowLine-0.8B-Gen1",
   "display_name": "KnowLine-0.8B-Gen1",
   "tag": "",
   "class_label": "Open · 0.8B",
   "base_model": "Qwen3.5-0.8B",
   "setup_label": "KnowLine-0.8B-Gen1",
   "maker": "PelaAI",
   "kind": "decision",
   "params_total": 0.8,
   "params_active": 0.8,
   "quantization": "BF16",
   "score_0_100": 10.251964863615349,
   "raw": {
    "metric": "accuracy",
    "value": 0.2820157189089228,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.10540986823469145,
   "improvement_points": null,
   "runs": [
    "math:knowline-0.8b_en",
    "math:knowline-0.8b_en_dontknow"
   ],
   "notes": "on our NVIDIA DGX Spark (GB10)",
   "gpu_hours": 0.017222222222222222,
   "gpu_cost_usd": 0.003665195893679951
  },
  {
   "repo": "knowledgator/gliclass-v4-edge",
   "commit": "885af1af8a07edf4ed34df83ca0d995ed8201283",
   "model_key": "gliclass-v4-edge",
   "name": "GLiClass v4 Edge",
   "display_name": "GLiClass v4 Edge",
   "tag": "",
   "class_label": "Open · 0.4B",
   "base_model": "DeBERTa-v3-large",
   "setup_label": "GLiClass v4 Edge",
   "maker": null,
   "kind": "decision",
   "params_total": 0.4,
   "params_active": 0.4,
   "quantization": "BF16",
   "score_0_100": 3.3171521035598692,
   "raw": {
    "metric": "accuracy",
    "value": 0.22653721682847897,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.07014710098192448,
   "improvement_points": null,
   "runs": [
    "math:gliclass-v4-edge_en",
    "math:gliclass-v4-edge_en_dontknow"
   ],
   "notes": "on our NVIDIA DGX Spark (GB10)",
   "gpu_hours": 0.008611111111111111,
   "gpu_cost_usd": 0.0018325979468399754
  },
  {
   "repo": "fastino/GLiDE-0.4B",
   "commit": "6bda7ab91aeb36dcad6d27828a8871e548c7fc71",
   "model_key": "glide-0.4b",
   "name": "GLiDE 0.4B",
   "display_name": "GLiDE 0.4B",
   "tag": "",
   "class_label": "Open · 0.4B",
   "base_model": null,
   "setup_label": "GLiDE 0.4B (open)",
   "maker": "Fastino",
   "kind": "decision",
   "params_total": 0.4,
   "params_active": 0.4,
   "quantization": "BF16",
   "score_0_100": 0.716597318539066,
   "raw": {
    "metric": "accuracy",
    "value": 0.20573277854831254,
    "questions": 2163,
    "options": 5
   },
   "calibration_error": 0.12157482885587954,
   "improvement_points": null,
   "runs": [
    "math:glide-0.4b_en",
    "math:glide-0.4b_en_dontknow"
   ],
   "notes": "on our NVIDIA DGX Spark (GB10)",
   "gpu_hours": 0.0077777777777777776,
   "gpu_cost_usd": 0.0016552497584361066
  }
 ]
}
