{
 "bench": "grading",
 "title": "Answer Check · does the text give this answer?",
 "url": "https://zekiworks.github.io/analysis/grading/",
 "updated": "2026-11-16T19:44:15Z",
 "score": {
  "zero": "uniform chance: one of the three labels at random on every pair (accuracy 1/3)",
  "hundred": "every one of the 2,069 pairs labelled right (accuracy 100%)"
 },
 "models": [
  {
   "name": "karpo-decision-31B",
   "display_name": "karpo-decision-31B",
   "tag": "",
   "model_key": "karpo-decision-31b",
   "maker": "MachinePulse",
   "kind": "decision",
   "repo": "MachinePulseAI/karpo-decision-31B",
   "commit": "d69cdd57eab23271cbe65236ca8a00d80d7bb8c7",
   "params_total": 31.3,
   "params_active": 31.3,
   "quantization": "BF16",
   "score_0_100": 89.99516674722086,
   "raw": {
    "metric": "accuracy",
    "value": 0.9333011116481392,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.02798799607249211,
   "improvement_points": null,
   "runs": [
    "grading:site-karpo-decision-31b-v1"
   ],
   "notes": "MachinePulseAI/karpo-decision-31B@d69cdd57, its card's karpo-decision package (machinepulse-ai/karpo-decision@ac40d399) System One server, BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 3 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
   "gpu_hours": 0.13244746838548202,
   "gpu_cost_usd": null
  },
  {
   "name": "Copernicus-31B",
   "display_name": "Copernicus-31B",
   "tag": "",
   "model_key": "copernicus-31b",
   "maker": "Poindexter Labs",
   "kind": "decision",
   "repo": "Poindexter-Labs/Copernicus-31B",
   "commit": "be715259e97a743341f2d7a3e6a6298dab23ca0e",
   "params_total": 31.3,
   "params_active": 31.3,
   "quantization": "BF16",
   "score_0_100": 88.9801836636056,
   "raw": {
    "metric": "accuracy",
    "value": 0.9265345577573707,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.10967730780108835,
   "improvement_points": null,
   "runs": [
    "grading:site-copernicus-31b-v1"
   ],
   "notes": "Poindexter-Labs/Copernicus-31B@be715259, its card's copernicus_server.py (Transformers backend, the card's defaults), BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 3 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
   "gpu_hours": 0.33160791070381207,
   "gpu_cost_usd": null
  },
  {
   "name": "KnowLine-31B-Gen1.5",
   "display_name": "KnowLine-31B-Gen1.5",
   "tag": "",
   "model_key": "knowline-31b-gen1.5",
   "maker": "PelaAI",
   "kind": "decision",
   "repo": "PelaAI/KnowLine-31B-Gen1.5",
   "commit": "8e56aa1b236a6dd1064dcfd22ba1e5e360e4c140",
   "params_total": 31.0,
   "params_active": 31.0,
   "quantization": "BF16",
   "score_0_100": 88.83518608023199,
   "raw": {
    "metric": "accuracy",
    "value": 0.9255679072015467,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.05716040274263797,
   "improvement_points": null,
   "runs": [
    "grading:site-knowline-31b-gen1.5-v1"
   ],
   "notes": "PelaAI/KnowLine-31B-Gen1.5@8e56aa1b, the card's serve_knowline.sh (SGLang), BF16, on one DGX Spark (ours) as for its published runs, self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 1 published-cohort pair was sent in test requests whose outputs were not kept, so its scored row comes from a later request",
   "gpu_hours": 1.5019916954069576,
   "gpu_cost_usd": null
  },
  {
   "name": "Sieve 31B v3",
   "display_name": "Sieve 31B v3",
   "tag": "",
   "model_key": "sieve-31b-v3",
   "maker": "Sthānika AI",
   "kind": "decision",
   "repo": "sthanika-ai/Sieve-31B-v3",
   "commit": "8fcee858f7f20a50bfa3c8f984c22adb3cbb46a2",
   "params_total": 31.0,
   "params_active": 31.0,
   "quantization": "BF16",
   "score_0_100": 88.11019816336393,
   "raw": {
    "metric": "accuracy",
    "value": 0.9207346544224263,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.0700594490091832,
   "improvement_points": null,
   "runs": [
    "grading:site-sieve-31b-v3-v1"
   ],
   "notes": "sthanika-ai/Sieve-31B-v3@8fcee858, sieve-serve 0.5.1 (text only, design 'letters'), BF16, on one DGX Spark (ours) as for its published runs, self-hosted; the server reports no input-token count; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 2 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
   "gpu_hours": 1.6228648508469032,
   "gpu_cost_usd": null
  },
  {
   "name": "KnowLine-31B-Gen2",
   "display_name": "KnowLine-31B-Gen2",
   "tag": "",
   "model_key": "knowline-31b-gen2",
   "maker": "PelaAI",
   "kind": "decision",
   "repo": "PelaAI/KnowLine-31B-Gen2",
   "commit": "fd99a9ed2255d8a67dd47a44cc429e638f611593",
   "params_total": 31.0,
   "params_active": 31.0,
   "quantization": "BF16",
   "score_0_100": 87.6752054132431,
   "raw": {
    "metric": "accuracy",
    "value": 0.917834702754954,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.07166560874597089,
   "improvement_points": null,
   "runs": [
    "grading:site-knowline-31b-gen2-v1"
   ],
   "notes": "PelaAI/KnowLine-31B-Gen2@fd99a9ed, its card's serve_knowline.sh (SGLang, then knowline_server.py, card defaults) in the lmsysorg/sglang image with SGLang 0.5.23 (the card names 0.5.21, which we do not have), BF16, SHA256SUMS checked, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); the run overlapped a host GPU recovery warning; its serving process continued, and the run completed without interruption or resume (producer report)",
   "gpu_hours": 0.2582788412532403,
   "gpu_cost_usd": null
  },
  {
   "name": "REAX 1.3",
   "display_name": "REAX 1.3",
   "tag": "",
   "model_key": "reax-1.3",
   "maker": "REAX",
   "kind": "decision",
   "repo": "decisionmodels/REAX-1.3",
   "commit": "fa4bf7ee3bb56a8bc421dad090631be1c5845c2d",
   "params_total": 57.36,
   "params_active": 57.36,
   "quantization": "FP8",
   "score_0_100": 86.95021749637505,
   "raw": {
    "metric": "accuracy",
    "value": 0.9130014499758338,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.048274518773097234,
   "improvement_points": null,
   "runs": [
    "grading:site-reax-1.3-v1"
   ],
   "notes": "decisionmodels/REAX-1.3@fa4bf7ee, REAX Engine reax13 server on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 3 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
   "gpu_hours": 0.42833667592136754,
   "gpu_cost_usd": null
  },
  {
   "name": "d1.2-ultra",
   "display_name": "d1.2-ultra",
   "tag": "",
   "model_key": "liquid-d1.2-ultra",
   "maker": "Liquid AI",
   "kind": "decision",
   "repo": null,
   "commit": null,
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 86.44272595456741,
   "raw": {
    "metric": "accuracy",
    "value": 0.9096181730304495,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.08786327021452686,
   "improvement_points": null,
   "runs": [
    "grading:site-liquid-d1.2-ultra-v1"
   ],
   "notes": "Liquid AI decisions API (api.liquid.ai/decisions, model d1.2-ultra); one question per request, 8 requests at a time (latency not comparable across different concurrency)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "name": "Turion Jev 31B",
   "display_name": "Turion Jev 31B",
   "tag": "",
   "model_key": "turion-jev-31b",
   "maker": "Turion.AI",
   "kind": "decision",
   "repo": "turion-ai/turion-jev-31b",
   "commit": "866c0fc6e040cba23fcc72113a80e5c99d5ebab1",
   "params_total": 31.0,
   "params_active": 31.0,
   "quantization": "FP8",
   "score_0_100": 86.44272595456741,
   "raw": {
    "metric": "accuracy",
    "value": 0.9096181730304495,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.06694250072498793,
   "improvement_points": null,
   "runs": [
    "grading:site-turion-jev-31b-v1"
   ],
   "notes": "turion-ai/turion-jev-31b@866c0fc6, the card's turion_jev.serve on vLLM 0.32.0 (BF16 weights, FP8 at load, text only), on one DGX Spark (ours) as for its published runs, self-hosted; vLLM memory share 0.80 instead of the card's 0.92 (KV cache only); one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 2 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
   "gpu_hours": 0.7637274554815272,
   "gpu_cost_usd": null
  },
  {
   "name": "Perplexity Decider 27B v1.3",
   "display_name": "Perplexity Decider 27B v1.3",
   "tag": "",
   "model_key": "perplexity-decider-27b-v1.3",
   "maker": "Perplexity",
   "kind": "decision",
   "repo": "perplexity-ai/pplx-decider-v1.3",
   "commit": "9c2ebb27470d3dec2ad7ab727727e40d142c7213",
   "params_total": 27.0,
   "params_active": 27.0,
   "quantization": "BF16",
   "score_0_100": 86.22522957950699,
   "raw": {
    "metric": "accuracy",
    "value": 0.9081681971967134,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.02761915331616796,
   "improvement_points": null,
   "runs": [
    "grading:site-perplexity-decider-27b-v1.3-v1"
   ],
   "notes": "perplexity-ai/pplx-decider-v1.3@9c2ebb27, the runner's pplx_decider_server.py with the checkpoint's own pplx_decider package (as for its published v1.2 runs), BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 3 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
   "gpu_hours": 0.14082144462229776,
   "gpu_cost_usd": null
  },
  {
   "name": "Inkling 2 Small",
   "display_name": "Inkling 2 Small",
   "tag": "thinking on (no off mode) · chat",
   "model_key": "inkling-2-small",
   "maker": "Thinking Machines",
   "kind": "chat",
   "repo": null,
   "commit": null,
   "params_total": 594.0,
   "params_active": 17.0,
   "quantization": null,
   "score_0_100": 86.22522957950699,
   "raw": {
    "metric": "accuracy",
    "value": 0.9081681971967134,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "grading:chat-inkling-2-small-v1"
   ],
   "notes": "OpenRouter thinkingmachines/inkling-2-small (paid per token; providers with no data retention and no training on inputs), reasoning at the route's default; billed per token at OpenRouter's list price; one question per request, 1 request at a time (latency not comparable across different concurrency); list prices $0.30 per 1M input and $1.00 per 1M output tokens (OpenRouter, 16 Nov); answered by OpenRouter's providers Together 1,104, Cloudflare 317, PrimeIntellect 242, Nebius 201, DeepInfra 179, Modal 26; OpenRouter's reported charges for the 2,069 billed requests of this model: $2.8921 (actual billing, distinct from the list-price figure); thinking on; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 0 of 2069 replies (0%) had no answer marker and count wrong; every reply kept",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "name": "GLiDE 27B",
   "display_name": "GLiDE 27B",
   "tag": "",
   "model_key": "glide-27b",
   "maker": "Fastino",
   "kind": "decision",
   "repo": "fastino/GLiDE-27B",
   "commit": "4d32743ae96b74f0116f050d37742bbd43c04e7d",
   "params_total": 27.0,
   "params_active": 27.0,
   "quantization": "BF16",
   "score_0_100": 86.0802319961334,
   "raw": {
    "metric": "accuracy",
    "value": 0.9072015466408894,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.03608106551078754,
   "improvement_points": null,
   "runs": [
    "grading:site-glide-27b-v1"
   ],
   "notes": "fastino/GLiDE-27B@4d32743a, GLiDE server (glide.server), BF16, GLIDE_MAX_BATCH=1, on one DGX Spark (ours) as for its published runs, self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 2 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
   "gpu_hours": 0.6924089500718801,
   "gpu_cost_usd": null
  },
  {
   "name": "Logits Sigma-1 Duo",
   "display_name": "Logits Sigma-1 Duo",
   "tag": "",
   "model_key": "logits-sigma-1-duo",
   "maker": "Logits",
   "kind": "decision",
   "repo": "logitshq/Logits-Sigma-1-Duo",
   "commit": "a0c1b6e8242916aafdcf9a5a49c1a952cb56e091",
   "params_total": 57.36,
   "params_active": 57.36,
   "quantization": "FP8",
   "score_0_100": 85.13774770420494,
   "raw": {
    "metric": "accuracy",
    "value": 0.9009183180280329,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.07037908459683984,
   "improvement_points": null,
   "runs": [
    "grading:site-logits-sigma-1-duo-v1"
   ],
   "notes": "logitshq/Logits-Sigma-1-Duo@a0c1b6e8 (weights and code identical to d9641cd8, the revision its card's run names; two members averaged per option, FP8 weights), its card's serving image (code/Dockerfile: torch 2.15, transformers 5.22.1, causal-conv1d 1.7.0, kit decision-index@25c6ec7) plus only gcc, libc6-dev and python3-dev, which its Triton kernels need to compile a launcher at first use and the image lacks, with the engine and option its serve.sh fixes (FP8 matmul on its own Triton kernel: its cuBLAS block-scaled path was not taken on this GPU, a fallback the engine makes itself), behind di_serve.py, our System One wrapper of the engine, on one RTX PRO 6000 (ours), self-hosted; text only (the load report lists FP8 scales missing for member A's vision tower, which text requests do not use; not checked on images); one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.41566869665931905,
   "gpu_cost_usd": null
  },
  {
   "name": "GLiDE",
   "display_name": "GLiDE",
   "tag": "API",
   "model_key": "fastino-glide",
   "maker": "Fastino",
   "kind": "decision",
   "repo": null,
   "commit": null,
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 85.06524891251811,
   "raw": {
    "metric": "accuracy",
    "value": 0.9004349927501208,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.02970154943215885,
   "improvement_points": null,
   "runs": [
    "grading:site-fastino-glide-v1"
   ],
   "notes": "Fastino decisions API (api.fastino.ai, model fastino/GLiDE); one question per request, 8 requests at a time (latency not comparable across different concurrency)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "name": "SubQ Snap 1.1",
   "display_name": "SubQ Snap 1.1",
   "tag": "",
   "model_key": "subq-snap-1.1",
   "maker": "Subquadratic",
   "kind": "decision",
   "repo": null,
   "commit": null,
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 84.92025132914449,
   "raw": {
    "metric": "accuracy",
    "value": 0.8994683421942967,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.02284575354194461,
   "improvement_points": null,
   "runs": [
    "grading:site-subq-snap-1.1-v1"
   ],
   "notes": "SubQ decisions API (api.subq.ai, model subq-snap-1.1); its probabilities were byte-identical to SubQ Snap Preview's on all 2,069 pairs; one question per request, 8 requests at a time (latency not comparable across different concurrency)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "name": "GLiDE",
   "display_name": "GLiDE",
   "tag": "v1.2",
   "model_key": "fastino-glide-v1.2",
   "maker": "Fastino",
   "kind": "decision",
   "repo": "fastino/GLiDE",
   "commit": "a7e45896be7c96ba3354891da790c5ec99c30b6c",
   "params_total": 27.0,
   "params_active": 27.0,
   "quantization": "BF16",
   "score_0_100": 84.92025132914449,
   "raw": {
    "metric": "accuracy",
    "value": 0.8994683421942967,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.04729810505945925,
   "improvement_points": null,
   "runs": [
    "grading:site-fastino-glide-v1.2-v1"
   ],
   "notes": "fastino/GLiDE@a7e4589, GLiDE server (Fastino's release code), BF16, GLIDE_MAX_BATCH=1, on one RTX PRO 6000 (ours) as the backgammon league's pinned server, self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 2 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
   "gpu_hours": 0.14576463736400147,
   "gpu_cost_usd": null
  },
  {
   "name": "d1-ultra",
   "display_name": "d1-ultra",
   "tag": "",
   "model_key": "liquid-d1-ultra",
   "maker": "Liquid AI",
   "kind": "decision",
   "repo": null,
   "commit": null,
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 84.70275495408409,
   "raw": {
    "metric": "accuracy",
    "value": 0.8980183663605606,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.11420895215834384,
   "improvement_points": null,
   "runs": [
    "grading:d1-ultra-v1"
   ],
   "notes": "Liquid AI decisions API (api.liquid.ai); one question per request, 8 requests at a time (latency not comparable across different concurrency)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "name": "Microsoft-Decision-1",
   "display_name": "Microsoft-Decision-1",
   "tag": "",
   "model_key": "microsoft-decision-1",
   "maker": "Microsoft",
   "kind": "decision",
   "repo": null,
   "commit": null,
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 83.90526824552923,
   "raw": {
    "metric": "accuracy",
    "value": 0.8927017883035283,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.023815195630622524,
   "improvement_points": null,
   "runs": [
    "grading:site-microsoft-decision-1-v1"
   ],
   "notes": "OpenRouter decisions API (openrouter.ai/api, model microsoft/microsoft-decision-1); one question per request, 8 requests at a time (latency not comparable across different concurrency)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "name": "vera-medium-omni",
   "display_name": "vera-medium-omni",
   "tag": "",
   "model_key": "vera-medium-omni",
   "maker": "scar-ai",
   "kind": "decision",
   "repo": "scar-ai/vera-medium-omni",
   "commit": "1f23a6cf5304e8e53b7ded53980765d5f8f4047c",
   "params_total": 4.2,
   "params_active": 4.2,
   "quantization": "BF16",
   "score_0_100": 83.68777187046882,
   "raw": {
    "metric": "accuracy",
    "value": 0.8912518124697921,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.12360604156597389,
   "improvement_points": null,
   "runs": [
    "grading:site-vera-medium-omni-v1"
   ],
   "notes": "scar-ai/vera-medium-omni@1f23a6cf, its card's own server (vera-serve from vera-s1 0.1.8, PyTorch engine), BF16, on one RTX PRO 6000 (ours), self-hosted; server probabilities are rounded to four decimal places and retained without renormalization; ECE uses the reported maximum probability; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.03423223886573668,
   "gpu_cost_usd": null
  },
  {
   "name": "Gemini 4 Hydrogen",
   "display_name": "Gemini 4 Hydrogen",
   "tag": "thinking on (no off mode) · chat",
   "model_key": "gemini-4-hydrogen",
   "maker": "Google",
   "kind": "chat",
   "repo": null,
   "commit": null,
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 83.68777187046882,
   "raw": {
    "metric": "accuracy",
    "value": 0.8912518124697921,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "grading:chat-gemini-4-hydrogen-v1"
   ],
   "notes": "Google Antigravity agent harness through Can's agbench bridge (Google AI Ultra subscription), thinking high, tool descriptions present and tools denied, a fresh conversation per request; $0 billed (the subscription's quota); one question per request, 1 request at a time (latency not comparable across different concurrency); the pass stopped once when the bridge's safety switch tripped on a false positive (fixed by its owner) and resumed in the same order by a recorded decision; the one request in flight at the stop was sent again, so its recorded outcome is not guaranteed to be the model's first output; thinking on; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 0 of 2069 replies (0%) had no answer marker and count wrong; every reply kept",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "name": "Jev 1.14",
   "display_name": "Jev 1.14",
   "tag": "",
   "model_key": "jev-1.14.0",
   "maker": "TypeSafe",
   "kind": "decision",
   "repo": null,
   "commit": null,
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 83.61527307878202,
   "raw": {
    "metric": "accuracy",
    "value": 0.8907684871918802,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.027994200096665072,
   "improvement_points": null,
   "runs": [
    "grading:site-jev-1.14.0-v1"
   ],
   "notes": "TypeSafe decisions API (api.typesafe.ai, model jev-1.14.0); one question per request, 8 requests at a time (latency not comparable across different concurrency)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "name": "Jev 1.13",
   "display_name": "Jev 1.13",
   "tag": "",
   "model_key": "jev-1.13.0",
   "maker": "TypeSafe",
   "kind": "decision",
   "repo": null,
   "commit": null,
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 82.96278395360076,
   "raw": {
    "metric": "accuracy",
    "value": 0.8864185596906718,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.018777187046882544,
   "improvement_points": null,
   "runs": [
    "grading:site-jev-1.13.0-v1"
   ],
   "notes": "TypeSafe decisions API (api.typesafe.ai, model jev-1.13.0); one question per request, 8 requests at a time (latency not comparable across different concurrency)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "name": "Jeffan-9B-1211",
   "display_name": "Jeffan-9B-1211",
   "tag": "",
   "model_key": "jeffan-9b-1211",
   "maker": "tattrongvu",
   "kind": "decision",
   "repo": "tattrongvu/Jeffan-9B-1211",
   "commit": "862d1a7a8b64a8d708ddb0b0cd775a740f01841a",
   "params_total": 9.0,
   "params_active": 9.0,
   "quantization": "BF16",
   "score_0_100": 82.96278395360076,
   "raw": {
    "metric": "accuracy",
    "value": 0.8864185596906718,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.033345923925473156,
   "improvement_points": null,
   "runs": [
    "grading:site-jeffan-9b-1211-v1"
   ],
   "notes": "tattrongvu/Jeffan-9B-1211@862d1a7a, its card's Decision Index engine (jeffan_engine:DecisionEngine, transformers 5.17.0, flash-linear-attention 0.5.2, kit decision-index@f8552a1, options max_len and budget 65,536 as its card; without the card's optional causal-conv1d, so its PyTorch reference convolution) behind di_serve.py, our System One wrapper of the engine, BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.05723418541718017,
   "gpu_cost_usd": null
  },
  {
   "name": "d1.1",
   "display_name": "d1.1",
   "tag": "",
   "model_key": "liquid-d1.1",
   "maker": "Liquid AI",
   "kind": "decision",
   "repo": null,
   "commit": null,
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 82.52779120347994,
   "raw": {
    "metric": "accuracy",
    "value": 0.8835186080231996,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.08687460160915417,
   "improvement_points": null,
   "runs": [
    "grading:site-liquid-d1.1-v1"
   ],
   "notes": "Liquid AI decisions API (api.liquid.ai/decisions, model d1.1); one question per request, 8 requests at a time (latency not comparable across different concurrency)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "name": "GLiDE 9B",
   "display_name": "GLiDE 9B",
   "tag": "",
   "model_key": "glide-9b",
   "maker": "Fastino",
   "kind": "decision",
   "repo": "fastino/GLiDE-9B",
   "commit": "ab54df5acba0eebf7df45560ffdaacd64e32df3b",
   "params_total": 7.94,
   "params_active": 7.94,
   "quantization": "BF16",
   "score_0_100": 82.2377960367327,
   "raw": {
    "metric": "accuracy",
    "value": 0.8815853069115515,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.07204617761476283,
   "improvement_points": null,
   "runs": [
    "grading:site-glide-9b-v1"
   ],
   "notes": "fastino/GLiDE-9B@ab54df5a, GLiDE server (python -m glide.server) in vLLM v0.30.0 image with flash-linear-attention 0.5.2, as for GLiDE 2B, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 3 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
   "gpu_hours": 0.05197667056112872,
   "gpu_cost_usd": null
  },
  {
   "name": "Cerno 9B v1.2",
   "display_name": "Cerno 9B v1.2",
   "tag": "",
   "model_key": "cerno-9b-v1.2",
   "maker": "Apertix AI",
   "kind": "decision",
   "repo": "apertixai/cerno-9b-v1.2",
   "commit": "f02a632fbe3a4698cd1ff5c0b929ea240e2b8eda",
   "params_total": 9.0,
   "params_active": 9.0,
   "quantization": "BF16",
   "score_0_100": 81.87530207829867,
   "raw": {
    "metric": "accuracy",
    "value": 0.8791686805219913,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.028972160463992265,
   "improvement_points": null,
   "runs": [
    "grading:site-cerno-9b-v1.2-v1"
   ],
   "notes": "apertixai/cerno-9b-v1.2@f02a632f, its card's Decision Index engine (CernoEngine, transformers 5.17.0, flash-linear-attention 0.5.2, kit decision-index@9eb2dbe2, the checkpoint's decider_config.json settings, loaded by Hub id and that revision as its card runs it, from the local copy; causal-conv1d not installed, so its PyTorch reference convolution) behind di_serve.py, our System One wrapper of the engine, BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.07257821582541932,
   "gpu_cost_usd": null
  },
  {
   "name": "StartLux-Decision-v1.1-4B-Instant",
   "display_name": "StartLux-Decision-v1.1-4B-Instant",
   "tag": "",
   "model_key": "startlux-decision-v1.1-4b",
   "maker": "StartLux",
   "kind": "decision",
   "repo": "startlux-models/StartLux-Decision-v1.1-4B-Instant",
   "commit": "416088f77cfce09633906a3675c7cd2e5dabcf10",
   "params_total": 4.0,
   "params_active": 4.0,
   "quantization": "BF16",
   "score_0_100": 81.65780570323827,
   "raw": {
    "metric": "accuracy",
    "value": 0.8777187046882552,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.08316936790980808,
   "improvement_points": null,
   "runs": [
    "grading:site-startlux-decision-v1.1-4b-v1"
   ],
   "notes": "startlux-models/StartLux-Decision-v1.1-4B-Instant@416088f7, its package's own server (startlux-decide serve --launch-vllm, StartLux-Decision@a99977d, vLLM 0.31.0) as for its published Reading and Logic & Math runs, BF16, on one DGX Spark (ours), self-hosted; licence CC-BY-NC-4.0 (research use); one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.08060698456742366,
   "gpu_cost_usd": null
  },
  {
   "name": "Dexy-Decision-9B-V2",
   "display_name": "Dexy-Decision-9B-V2",
   "tag": "",
   "model_key": "dexy-decision-9b-v2",
   "maker": "Patrick-18",
   "kind": "decision",
   "repo": "Patrick-18/Dexy-Decision-9B-V2",
   "commit": "4f434e2b5fd9f0addfe3233489571c2dc8ce7a19",
   "params_total": 9.0,
   "params_active": 9.0,
   "quantization": "BF16",
   "score_0_100": 81.07781536974382,
   "raw": {
    "metric": "accuracy",
    "value": 0.8738521024649589,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.025398751062972964,
   "improvement_points": null,
   "runs": [
    "grading:site-dexy-decision-9b-v2-v1"
   ],
   "notes": "Patrick-18/Dexy-Decision-9B-V2@4f434e2b, its card's Transformers service (d3_server.py, in the environment its requirements.txt pins: torch 2.14.0, transformers 5.19.0), BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.04106403172565293,
   "gpu_cost_usd": null
  },
  {
   "name": "KnowLine-12B-Gen1",
   "display_name": "KnowLine-12B-Gen1",
   "tag": "",
   "model_key": "knowline-12b-gen1",
   "maker": "PelaAI",
   "kind": "decision",
   "repo": "PelaAI/KnowLine-12B-Gen1",
   "commit": "f26fbc2d64092bb2f8caaedbec243172bb3e908f",
   "params_total": 12.0,
   "params_active": 12.0,
   "quantization": "BF16",
   "score_0_100": 80.86031899468341,
   "raw": {
    "metric": "accuracy",
    "value": 0.8724021266312229,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.06837137729635548,
   "improvement_points": null,
   "runs": [
    "grading:site-knowline-12b-gen1-v1"
   ],
   "notes": "PelaAI/KnowLine-12B-Gen1@f26fbc2d, its card's serve_knowline.sh (SGLang, then knowline_server.py, card defaults) in the lmsysorg/sglang image with SGLang 0.5.23 (the card names 0.5.21, which we do not have), BF16, SHA256SUMS checked, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.11182212392749963,
   "gpu_cost_usd": null
  },
  {
   "name": "KnowLine 4B Gen5.5",
   "display_name": "KnowLine 4B Gen5.5",
   "tag": "",
   "model_key": "knowline-4b-gen5.5",
   "maker": "PelaAI",
   "kind": "decision",
   "repo": "PelaAI/KnowLine-4B-Gen5.5",
   "commit": "779944d2cc8ffa19cef386aee83c83bcb967ade2",
   "params_total": 4.0,
   "params_active": 4.0,
   "quantization": "FP8",
   "score_0_100": 80.642822619623,
   "raw": {
    "metric": "accuracy",
    "value": 0.8709521507974867,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.045316565330469655,
   "improvement_points": null,
   "runs": [
    "grading:site-knowline-4b-gen5.5-v1"
   ],
   "notes": "PelaAI/KnowLine-4B-Gen5.5@779944d2 (the site's pin; no longer in the maker's repository, copied from the snapshot its published runs used, SHA256SUMS checked), its card's serve_knowline.sh (SGLang with FP8 at load, then knowline_server.py, card defaults) in the lmsysorg/sglang image with SGLang 0.5.23, on one RTX PRO 6000 (ours), self-hosted; its published runs were on a DGX Spark; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.028165908471719098,
   "gpu_cost_usd": null
  },
  {
   "name": "Adjudex 9B v5",
   "display_name": "Adjudex 9B v5",
   "tag": "",
   "model_key": "adjudex-9b-v5",
   "maker": "newsbang",
   "kind": "decision",
   "repo": "newsbang/adjudex-9b-v5",
   "commit": "ffdcb8254d1b22e40b2bbbf569f143e0b357aeca",
   "params_total": 9.0,
   "params_active": 9.0,
   "quantization": "BF16",
   "score_0_100": 80.42532624456258,
   "raw": {
    "metric": "accuracy",
    "value": 0.8695021749637506,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.049161628998359276,
   "improvement_points": null,
   "runs": [
    "grading:site-adjudex-9b-v5-v1"
   ],
   "notes": "newsbang/adjudex-9b-v5@ffdcb825, its card's autojev server (the Perplexity Decider v1.1 source, in the environment of its requirements), default settings, BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.04104341335131996,
   "gpu_cost_usd": null
  },
  {
   "name": "MiniCPM5-2B",
   "display_name": "MiniCPM5-2B",
   "tag": "thinking on · chat",
   "model_key": "minicpm5-2b",
   "maker": "OpenBMB",
   "kind": "chat",
   "repo": "openbmb/MiniCPM5-2B",
   "commit": "42433e90dad95123ab87729fd7011e7110629556",
   "params_total": 2.52,
   "params_active": 2.52,
   "quantization": "BF16",
   "score_0_100": 79.2411793136781,
   "raw": {
    "metric": "accuracy",
    "value": 0.8616078620911873,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "grading:chat-minicpm5-2b-seed0-r2-v1",
    "grading:chat-minicpm5-2b-seed1-r2-v1",
    "grading:chat-minicpm5-2b-seed2-r2-v1"
   ],
   "notes": "openbmb/MiniCPM5-2B@42433e90, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 30 requests at a time from the client (latency not comparable across different concurrency); mean of 3 full runs, seeds 0, 1, 2 (min-max across seeds, not a confidence interval); answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; replies without an answer marker count wrong: mean rate 0.06% (per seed in seed_runs; cut off at the token limit per seed: 0, 0, 0); the three runs are fresh runs (14 Nov) with every reply kept, with the same model, sampling settings and requests at a time as the runs they replace and the server settings recorded for them; the earlier runs stay in history; gpu_hours is the mean of the three runs",
   "gpu_hours": 0.11305822975990755,
   "gpu_cost_usd": null,
   "accuracy_basis": "mean of 3 seeds",
   "seed_min": 0.8598356694055099,
   "seed_max": 0.8637022716288062,
   "seed_runs": [
    {
     "run": "grading:chat-minicpm5-2b-seed0-r2-v1",
     "seed": 0,
     "accuracy": 0.8598356694055099,
     "invalid": 3,
     "cutoff": 0,
     "wrong_answers_caught": 250
    },
    {
     "run": "grading:chat-minicpm5-2b-seed1-r2-v1",
     "seed": 1,
     "accuracy": 0.861285645239246,
     "invalid": 0,
     "cutoff": 0,
     "wrong_answers_caught": 258
    },
    {
     "run": "grading:chat-minicpm5-2b-seed2-r2-v1",
     "seed": 2,
     "accuracy": 0.8637022716288062,
     "invalid": 1,
     "cutoff": 0,
     "wrong_answers_caught": 256
    }
   ],
   "invalid_rate_mean": 0.000644433703882713
  },
  {
   "name": "d1",
   "display_name": "d1",
   "tag": "",
   "model_key": "liquid-d1",
   "maker": "Liquid AI",
   "kind": "decision",
   "repo": null,
   "commit": null,
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 79.19284678588689,
   "raw": {
    "metric": "accuracy",
    "value": 0.861285645239246,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.03076448878623012,
   "improvement_points": null,
   "runs": [
    "grading:site-liquid-d1-v1"
   ],
   "notes": "Liquid AI decisions API (api.liquid.ai/decisions, model d1); billed 'd1' route, the site's 'd1:free' was withdrawn (HTTP 404, 13 Nov); one question per request, 8 requests at a time (latency not comparable across different concurrency)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "name": "Perplexity Decider Flash v1.3",
   "display_name": "Perplexity Decider Flash v1.3",
   "tag": "",
   "model_key": "perplexity-decider-flash-v1.3",
   "maker": "Perplexity",
   "kind": "decision",
   "repo": "perplexity-ai/pplx-decider-flash-v1.3",
   "commit": "d06fb3f2d97e0a52325a0d463c450cf0338f44fe",
   "params_total": 4.0,
   "params_active": 4.0,
   "quantization": "BF16",
   "score_0_100": 78.83035282745286,
   "raw": {
    "metric": "accuracy",
    "value": 0.8588690188496858,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.041819332198274745,
   "improvement_points": null,
   "runs": [
    "grading:site-perplexity-decider-flash-v1.3-v1"
   ],
   "notes": "perplexity-ai/pplx-decider-flash-v1.3@d06fb3f2, the runner's pplx_decider_server.py with the checkpoint's own pplx_decider package (as for its published v1.2 runs), BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 3 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
   "gpu_hours": 0.035180532006086604,
   "gpu_cost_usd": null
  },
  {
   "name": "StartLux-Decision-v1.1-9B-Instant",
   "display_name": "StartLux-Decision-v1.1-9B-Instant",
   "tag": "",
   "model_key": "startlux-decision-v1.1-9b",
   "maker": "StartLux",
   "kind": "decision",
   "repo": "startlux-models/StartLux-Decision-v1.1-9B-Instant",
   "commit": "18a17f734966b213b2f00fa9c915e8b9af2a9b20",
   "params_total": 9.0,
   "params_active": 9.0,
   "quantization": "BF16",
   "score_0_100": 78.83035282745286,
   "raw": {
    "metric": "accuracy",
    "value": 0.8588690188496858,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.0823722971203353,
   "improvement_points": null,
   "runs": [
    "grading:site-startlux-decision-v1.1-9b-v1"
   ],
   "notes": "startlux-models/StartLux-Decision-v1.1-9B-Instant@18a17f73, its package's own server (startlux-decide serve --launch-vllm, StartLux-Decision@a99977d, vLLM 0.31.0), BF16, on one DGX Spark (ours), self-hosted; licence CC-BY-NC-4.0 (research use); one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.15375955299852695,
   "gpu_cost_usd": null
  },
  {
   "name": "Chinese Decision Pointer 4B",
   "display_name": "Chinese Decision Pointer 4B",
   "tag": "",
   "model_key": "chinese-decision-pointer-4b",
   "maker": "quincyqiang",
   "kind": "decision",
   "repo": "quincyqiang/chinese-decision-pointer-4b",
   "commit": "889f23e2655e5cba1208bb17f90526092813f5f2",
   "params_total": 4.0,
   "params_active": 4.0,
   "quantization": "BF16",
   "score_0_100": 78.10536491058483,
   "raw": {
    "metric": "accuracy",
    "value": 0.8540357660705655,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.0666280824070461,
   "improvement_points": null,
   "runs": [
    "grading:site-chinese-decision-pointer-4b-v1"
   ],
   "notes": "quincyqiang/chinese-decision-pointer-4b@889f23e2, its card's Decision Index engine (yanqiangmiffy/chinese-decision@c16450b, ChineseDecisionEngine with debias 'reverse' as its card's run; transformers 5.17.0, kit decision-index@9eb2dbe2, as the card names no kit commit; causal-conv1d not installed, so its PyTorch reference convolution) behind di_serve.py, our System One wrapper of the engine, BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.04414093524818924,
   "gpu_cost_usd": null
  },
  {
   "name": "Perplexity Decider Flash v1.2",
   "display_name": "Perplexity Decider Flash v1.2",
   "tag": "",
   "model_key": "perplexity-decider-flash-v1.2",
   "maker": "Perplexity",
   "kind": "decision",
   "repo": "perplexity-ai/pplx-decider-flash-v1.2",
   "commit": "6d771ad7b5ecff5e93b8fc7b90d4c5ccc341954d",
   "params_total": 4.0,
   "params_active": 4.0,
   "quantization": "BF16",
   "score_0_100": 77.16288061865636,
   "raw": {
    "metric": "accuracy",
    "value": 0.8477525374577091,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.02793073152029759,
   "improvement_points": null,
   "runs": [
    "grading:site-perplexity-decider-flash-v1.2-v1"
   ],
   "notes": "perplexity-ai/pplx-decider-flash-v1.2@6d771ad7, the runner's pplx_decider_server.py with the checkpoint's own pplx_decider package (as for its published v1.2 runs), BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.037100849536345196,
   "gpu_cost_usd": null
  },
  {
   "name": "Gemma 4 E4B",
   "display_name": "Gemma 4 E4B",
   "tag": "thinking on · chat",
   "model_key": "gemma-4-e4b-think",
   "maker": "Google",
   "kind": "chat",
   "repo": "google/gemma-4-E4B-it",
   "commit": "4ecc2ba63bcac23ba4a767fc87025b1450f84c9c",
   "params_total": 8.0,
   "params_active": 8.0,
   "quantization": "BF16",
   "score_0_100": 77.16288061865636,
   "raw": {
    "metric": "accuracy",
    "value": 0.8477525374577091,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "grading:chat-gemma-4-e4b-think-v1"
   ],
   "notes": "google/gemma-4-E4B-it@4ecc2ba6, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 30 requests at a time (latency not comparable across different concurrency); thinking on; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 2 of 2069 replies (0.10%) had no answer marker and count wrong (0 of them cut off at the 32,768-token limit); every reply kept; an earlier attempt of this same run stopped part-way (510 recorded pairs and an unknown number of further requests possibly answered but not recorded) and is kept apart, not merged; this attempt started again from the first pair, so its outcomes are this attempt's recorded outputs, not guaranteed to be the model's first outputs on those pairs",
   "gpu_hours": 0.18707697313942698,
   "gpu_cost_usd": null
  },
  {
   "name": "Gemini Decision 1",
   "display_name": "Gemini Decision 1",
   "tag": "",
   "model_key": "gemini-decision-1",
   "maker": "Google",
   "kind": "decision",
   "repo": null,
   "commit": null,
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 77.01788303528274,
   "raw": {
    "metric": "accuracy",
    "value": 0.8467858869018849,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.03889325895040145,
   "improvement_points": null,
   "runs": [
    "grading:gemini-v1"
   ],
   "notes": "Google Decisions API (generativelanguage.googleapis.com, model gemini-decision-1); one question per request, 8 requests at a time (latency not comparable across different concurrency)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "name": "Gemma 4 12B Unified",
   "display_name": "Gemma 4 12B Unified",
   "tag": "thinking off · chat",
   "model_key": "gemma-4-12b",
   "maker": "Google",
   "kind": "chat",
   "repo": "google/gemma-4-12B-it",
   "commit": "43c9d014b231474fa7212dbb48a762566c2b255d",
   "params_total": 11.95,
   "params_active": 11.95,
   "quantization": "BF16",
   "score_0_100": 77.01788303528274,
   "raw": {
    "metric": "accuracy",
    "value": 0.8467858869018849,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "grading:chat-gemma-4-12b-r2-v1"
   ],
   "notes": "google/gemma-4-12B-it@43c9d014, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time (latency not comparable across different concurrency); thinking off; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 0 of 2069 replies (0%) had no answer marker and count wrong; a fresh run (14-15 Nov) with every reply kept, with the same model, sampling settings and requests at a time as the run it replaces and a server from the same launcher (the earlier run's own server flags were not recorded); the earlier run stays in history",
   "gpu_hours": 0.06945622949589676,
   "gpu_cost_usd": null
  },
  {
   "name": "Elephant-1",
   "display_name": "Elephant-1",
   "tag": "",
   "model_key": "elephant-1",
   "maker": "Base Labs",
   "kind": "decision",
   "repo": "baselabs/elephant-1",
   "commit": "b3f8a93462fb1051a9de031e94a7ff380e254072",
   "params_total": 9.0,
   "params_active": 9.0,
   "quantization": "BF16",
   "score_0_100": 76.87288545190913,
   "raw": {
    "metric": "accuracy",
    "value": 0.8458192363460609,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.03695449589173516,
   "improvement_points": null,
   "runs": [
    "grading:site-elephant-1-v1"
   ],
   "notes": "baselabs/elephant-1@b3f8a934, the runner's baselabs_server.py in the card's setup.sh environment (OneJev 81ce62f1 with the checkpoint's serve.patch), on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency); before the run, 3 published-cohort pairs were sent in test requests whose outputs were not kept, so their scored rows come from a later request",
   "gpu_hours": 0.056309502522205596,
   "gpu_cost_usd": null
  },
  {
   "name": "Winnow-12B",
   "display_name": "Winnow-12B",
   "tag": "",
   "model_key": "winnow-12b",
   "maker": "EldanRing",
   "kind": "decision",
   "repo": "EldanRing/Winnow-12B",
   "commit": "b2b14213dfa252e6d6b543c8b334762e51772d29",
   "params_total": 12.0,
   "params_active": 12.0,
   "quantization": "Q8_0",
   "score_0_100": 76.58289028516192,
   "raw": {
    "metric": "accuracy",
    "value": 0.8438859352344128,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.08255692633693724,
   "improvement_points": null,
   "runs": [
    "grading:site-winnow-12b-v1"
   ],
   "notes": "EldanRing/Winnow-12B@b2b14213, its Q8_0 GGUF served by winnow-inference@d4631fb, the server its Logic & Math runs use, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.07591675669531456,
   "gpu_cost_usd": null
  },
  {
   "name": "KnowLine-9B-Gen1",
   "display_name": "KnowLine-9B-Gen1",
   "tag": "",
   "model_key": "knowline-9b-gen1",
   "maker": "PelaAI",
   "kind": "decision",
   "repo": "PelaAI/KnowLine-9B-Gen1",
   "commit": "325378d128e9f5eb5088196668f5874c5682b6c0",
   "params_total": 9.0,
   "params_active": 9.0,
   "quantization": "BF16",
   "score_0_100": 76.14789753504107,
   "raw": {
    "metric": "accuracy",
    "value": 0.8409859835669405,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.035704499480152495,
   "improvement_points": null,
   "runs": [
    "grading:site-knowline-9b-gen1-v1"
   ],
   "notes": "PelaAI/KnowLine-9B-Gen1@325378d1, its card's serve_knowline.sh (SGLang, then knowline_server.py, card defaults) in the lmsysorg/sglang image with SGLang 0.5.23 (the card's release launcher names 0.5.21, which we do not have), BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.06016774425950634,
   "gpu_cost_usd": null
  },
  {
   "name": "Granite-4.2-3B",
   "display_name": "Granite-4.2-3B",
   "tag": "thinking on · chat",
   "model_key": "granite-4.2-3b-think",
   "maker": "IBM",
   "kind": "chat",
   "repo": "ibm-granite/granite-4.2-3b",
   "commit": "4e9c275e232f10211ec88644d106a1f6a1b60e06",
   "params_total": 3.0,
   "params_active": 3.0,
   "quantization": "BF16",
   "score_0_100": 75.85790236829386,
   "raw": {
    "metric": "accuracy",
    "value": 0.8390526824552924,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "grading:chat-granite-4.2-3b-think-v1"
   ],
   "notes": "ibm-granite/granite-4.2-3b@4e9c275e, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 30 requests at a time (latency not comparable across different concurrency); thinking on; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 0 of 2069 replies (0%) had no answer marker and count wrong; every reply kept",
   "gpu_hours": 0.603654108024119,
   "gpu_cost_usd": null
  },
  {
   "name": "LFM3-1.7B",
   "display_name": "LFM3-1.7B",
   "tag": "thinking on · chat",
   "model_key": "lfm3-1.7b",
   "maker": "Liquid AI",
   "kind": "chat",
   "repo": "LiquidAI/LFM3-1.7B",
   "commit": "0dc4268b561499c648c7ba7c9466bf3d8d01517a",
   "params_total": 1.73,
   "params_active": 1.73,
   "quantization": "BF16",
   "score_0_100": 75.78540357660704,
   "raw": {
    "metric": "accuracy",
    "value": 0.8385693571773803,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "grading:chat-lfm3-1.7b-seed0-replay-v1",
    "grading:chat-lfm3-1.7b-seed1-r2-v1",
    "grading:chat-lfm3-1.7b-seed2-r2-v1"
   ],
   "notes": "LiquidAI/LFM3-1.7B@0dc4268b, BF16, cu135-nightly-be47189f37f6162991366ff6b91539f6e9d1f432 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 30 requests at a time from the client (latency not comparable across different concurrency); mean of 3 full runs, seeds 0, 1, 2 (min-max across seeds, not a confidence interval); answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; replies without an answer marker count wrong: mean rate 0.06% (per seed in seed_runs; cut off at the token limit per seed: 2, 1, 1); the three runs are fresh runs (14 Nov) with every reply kept, with the same model, sampling settings and requests at a time as the runs they replace and the server settings recorded for them; seed 0 is the rerun made to diagnose the missing answer lines; the earlier runs stay in history; gpu_hours is the mean of the three runs",
   "gpu_hours": 0.15417772855343545,
   "gpu_cost_usd": null,
   "accuracy_basis": "mean of 3 seeds",
   "seed_min": 0.832769453842436,
   "seed_max": 0.8434026099565007,
   "seed_runs": [
    {
     "run": "grading:chat-lfm3-1.7b-seed0-replay-v1",
     "seed": 0,
     "accuracy": 0.8395360077332045,
     "invalid": 2,
     "cutoff": 2,
     "wrong_answers_caught": 215
    },
    {
     "run": "grading:chat-lfm3-1.7b-seed1-r2-v1",
     "seed": 1,
     "accuracy": 0.832769453842436,
     "invalid": 1,
     "cutoff": 1,
     "wrong_answers_caught": 194
    },
    {
     "run": "grading:chat-lfm3-1.7b-seed2-r2-v1",
     "seed": 2,
     "accuracy": 0.8434026099565007,
     "invalid": 1,
     "cutoff": 1,
     "wrong_answers_caught": 216
    }
   ],
   "invalid_rate_mean": 0.000644433703882713
  },
  {
   "name": "Verdin-4B-v1.1",
   "display_name": "Verdin-4B-v1.1",
   "tag": "",
   "model_key": "verdin-4b-v1.1",
   "maker": "victorbennett",
   "kind": "decision",
   "repo": "victorbennett/Verdin-4B-v1.1",
   "commit": "45830d216bb7e45f43f0a482c131a7cafa6d189f",
   "params_total": 4.0,
   "params_active": 4.0,
   "quantization": "BF16",
   "score_0_100": 75.35041082648621,
   "raw": {
    "metric": "accuracy",
    "value": 0.8356694055099082,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.03399785843308816,
   "improvement_points": null,
   "runs": [
    "grading:site-verdin-4b-v1.1-v1"
   ],
   "notes": "victorbennett/Verdin-4B-v1.1@45830d21, its card's own server (python -m verdin.server, in the package versions its card lists: torch 2.8.0 cu128, transformers 5.17.0, flash-linear-attention 0.5.2, triton 3.8.0; no causal-conv1d, so its own torch conv path), BF16, on one RTX PRO 6000 (ours), self-hosted; licence CC-BY-NC-4.0 (research use); one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.031078735832046005,
   "gpu_cost_usd": null
  },
  {
   "name": "KnowLine-9B-Gen2",
   "display_name": "KnowLine-9B-Gen2",
   "tag": "",
   "model_key": "knowline-9b-gen2",
   "maker": "PelaAI",
   "kind": "decision",
   "repo": "PelaAI/KnowLine-9B-Gen2",
   "commit": "8ec8eb609bc10aa6050ae3754aee419d458fae42",
   "params_total": 9.0,
   "params_active": 9.0,
   "quantization": "BF16",
   "score_0_100": 75.27791203479943,
   "raw": {
    "metric": "accuracy",
    "value": 0.8351860802319961,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.04495010483366471,
   "improvement_points": null,
   "runs": [
    "grading:site-knowline-9b-gen2-v1"
   ],
   "notes": "PelaAI/KnowLine-9B-Gen2@8ec8eb60, its card's serve_knowline.sh (SGLang, then knowline_server.py, card defaults) in the lmsysorg/sglang image with SGLang 0.5.23 (the card names 0.5.21, which we do not have), BF16, SHA256SUMS checked, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.07386159999956078,
   "gpu_cost_usd": null
  },
  {
   "name": "Gemma 4 E4B",
   "display_name": "Gemma 4 E4B",
   "tag": "thinking on · sampled (temperature 1.0) · chat",
   "model_key": "gemma-4-e4b-think-sampled",
   "maker": "Google",
   "kind": "chat",
   "repo": "google/gemma-4-E4B-it",
   "commit": "4ecc2ba63bcac23ba4a767fc87025b1450f84c9c",
   "params_total": 8.0,
   "params_active": 8.0,
   "quantization": "BF16",
   "score_0_100": 75.1087481875302,
   "raw": {
    "metric": "accuracy",
    "value": 0.8340583212502014,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "grading:chat-gemma-4-e4b-think-sampled-seed0-v1",
    "grading:chat-gemma-4-e4b-think-sampled-seed1-v1",
    "grading:chat-gemma-4-e4b-think-sampled-seed2-v1"
   ],
   "notes": "google/gemma-4-E4B-it@4ecc2ba6, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 30 requests at a time from the client (latency not comparable across different concurrency); mean of 3 full runs, seeds 0, 1, 2 (min-max across seeds, not a confidence interval); answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; replies without an answer marker count wrong: mean rate 2% (per seed in seed_runs; cut off at the token limit per seed: 0, 0, 0); three new runs (16 Nov, every reply kept), seeds 0, 1, 2 at Gemma's own sampling (its generation_config.json: temperature 1.0, top_p 0.95, top_k 64; Ümit #10554), with the weights, template and token limit of the temperature-0 row, which stays as its own row; their server settings recorded; gpu_hours is the mean of the three runs",
   "gpu_hours": 0.13427235311653798,
   "gpu_cost_usd": null,
   "accuracy_basis": "mean of 3 seeds",
   "seed_min": 0.8274528757854036,
   "seed_max": 0.8395360077332045,
   "seed_runs": [
    {
     "run": "grading:chat-gemma-4-e4b-think-sampled-seed0-v1",
     "seed": 0,
     "accuracy": 0.8395360077332045,
     "invalid": 34,
     "cutoff": 0,
     "wrong_answers_caught": 216
    },
    {
     "run": "grading:chat-gemma-4-e4b-think-sampled-seed1-v1",
     "seed": 1,
     "accuracy": 0.8274528757854036,
     "invalid": 31,
     "cutoff": 0,
     "wrong_answers_caught": 208
    },
    {
     "run": "grading:chat-gemma-4-e4b-think-sampled-seed2-v1",
     "seed": 2,
     "accuracy": 0.8351860802319961,
     "invalid": 30,
     "cutoff": 0,
     "wrong_answers_caught": 211
    }
   ],
   "invalid_rate_mean": 0.015305300467214436
  },
  {
   "name": "LFM3-1.7B",
   "display_name": "LFM3-1.7B",
   "tag": "thinking off · chat",
   "model_key": "lfm3-1.7b-nothink",
   "maker": "Liquid AI",
   "kind": "chat",
   "repo": "LiquidAI/LFM3-1.7B",
   "commit": "0dc4268b561499c648c7ba7c9466bf3d8d01517a",
   "params_total": 1.73,
   "params_active": 1.73,
   "quantization": "BF16",
   "score_0_100": 74.89125181246979,
   "raw": {
    "metric": "accuracy",
    "value": 0.8326083454164653,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "grading:chat-lfm3-1.7b-nothink-r2-v1",
    "grading:chat-lfm3-1.7b-nothink-seed1-r2-v1",
    "grading:chat-lfm3-1.7b-nothink-seed2-r2-v1"
   ],
   "notes": "LiquidAI/LFM3-1.7B@0dc4268b, BF16, cu135-nightly-be47189f37f6162991366ff6b91539f6e9d1f432 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time from the client (latency not comparable across different concurrency); mean of 3 full runs, seeds 0, 1, 2 (min-max across seeds, not a confidence interval); answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; replies without an answer marker count wrong: mean rate 0.08% (per seed in seed_runs; cut off at the token limit per seed: 2, 2, 1); seed 0 is the published single run (14 Nov, every reply kept); seeds 1 and 2 are new runs (15 Nov, every reply kept) with the same model, sampling settings, requests at a time and server launcher, only the seed changed, their server settings recorded; gpu_hours is the mean of the three runs",
   "gpu_hours": 0.10339265255892785,
   "gpu_cost_usd": null,
   "accuracy_basis": "mean of 3 seeds",
   "seed_min": 0.8231029482841953,
   "seed_max": 0.8376027066215563,
   "seed_runs": [
    {
     "run": "grading:chat-lfm3-1.7b-nothink-r2-v1",
     "seed": 0,
     "accuracy": 0.8376027066215563,
     "invalid": 2,
     "cutoff": 2,
     "wrong_answers_caught": 212
    },
    {
     "run": "grading:chat-lfm3-1.7b-nothink-seed1-r2-v1",
     "seed": 1,
     "accuracy": 0.8371193813436443,
     "invalid": 2,
     "cutoff": 2,
     "wrong_answers_caught": 209
    },
    {
     "run": "grading:chat-lfm3-1.7b-nothink-seed2-r2-v1",
     "seed": 2,
     "accuracy": 0.8231029482841953,
     "invalid": 1,
     "cutoff": 1,
     "wrong_answers_caught": 197
    }
   ],
   "invalid_rate_mean": 0.0008055421298533914
  },
  {
   "name": "Valen-Decision-1.5-9B",
   "display_name": "Valen-Decision-1.5-9B",
   "tag": "",
   "model_key": "valen-decision-1.5-9b",
   "maker": "Valen-Team",
   "kind": "decision",
   "repo": "Valen-Team/Valen-Decision-1.5-9B",
   "commit": "5a512117ebc0f2b69f311becfcf47cdef8abf568",
   "params_total": 9.0,
   "params_active": 9.0,
   "quantization": "BF16",
   "score_0_100": 74.26292895118414,
   "raw": {
    "metric": "accuracy",
    "value": 0.8284195263412276,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.03150234591802053,
   "improvement_points": null,
   "runs": [
    "grading:site-valen-decision-1.5-9b-v1"
   ],
   "notes": "Valen-Team/Valen-Decision-1.5-9B@5a512117, its card's vLLM deployment (valen-vllm, the Valen plugin at Valen-Team/Valen@773d3a64 in the official vllm/vllm-openai:v0.33.0 image), BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.04507349295907858,
   "gpu_cost_usd": null
  },
  {
   "name": "Cygnet",
   "display_name": "Cygnet",
   "tag": "",
   "model_key": "cygnet",
   "maker": "Blockbrain Labs",
   "kind": "decision",
   "repo": null,
   "commit": null,
   "params_total": 11.95,
   "params_active": 11.95,
   "quantization": "BF16",
   "score_0_100": 72.88545190913484,
   "raw": {
    "metric": "accuracy",
    "value": 0.819236346060899,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.06509097176030178,
   "improvement_points": null,
   "runs": [
    "grading:site-cygnet-v1"
   ],
   "notes": "no weights of its own: google/gemma-4-12B-it@707f0a3b, unchanged, on vLLM 0.30.0 behind cygnet-recipe's shim (blockbrain-ai/cygnet-recipe@3cf591c, temperature 3.4), the server its Logic & Math runs use, BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.05341540666628283,
   "gpu_cost_usd": null
  },
  {
   "name": "Gemma 4 E2B",
   "display_name": "Gemma 4 E2B",
   "tag": "thinking on · chat",
   "model_key": "gemma-4-e2b-think",
   "maker": "Google",
   "kind": "chat",
   "repo": "google/gemma-4-E2B-it",
   "commit": "4adf15748de9e1a0097b6274cee9c15836c41c37",
   "params_total": 5.1,
   "params_active": 5.1,
   "quantization": "BF16",
   "score_0_100": 71.14548090865152,
   "raw": {
    "metric": "accuracy",
    "value": 0.8076365393910101,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "grading:chat-gemma-4-e2b-think-r2-v1"
   ],
   "notes": "google/gemma-4-E2B-it@4adf1574, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time (latency not comparable across different concurrency); thinking on; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 0 of 2069 replies (0%) had no answer marker and count wrong; a fresh run (14-15 Nov) with every reply kept, with the same model, sampling settings and requests at a time as the run it replaces and a server from the same launcher (the earlier run's own server flags were not recorded); the earlier run stays in history",
   "gpu_hours": 0.039903491282575095,
   "gpu_cost_usd": null
  },
  {
   "name": "Qwen3.5-2B",
   "display_name": "Qwen3.5-2B",
   "tag": "thinking on · chat",
   "model_key": "qwen3.5-2b",
   "maker": "Alibaba",
   "kind": "chat",
   "repo": "Qwen/Qwen3.5-2B",
   "commit": "15852e8c16360a2fea060d615a32b45270f8a8fc",
   "params_total": 2.27,
   "params_active": 2.27,
   "quantization": "BF16",
   "score_0_100": 70.34799420009666,
   "raw": {
    "metric": "accuracy",
    "value": 0.8023199613339778,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "grading:chat-qwen3.5-2b-r2-v1"
   ],
   "notes": "Qwen/Qwen3.5-2B@15852e8c, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time (latency not comparable across different concurrency); thinking on; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 78 of 2069 replies (4%) had no answer marker and count wrong (5 of them cut off at the 32,768-token limit); a fresh run (14-15 Nov) with every reply kept, with the same model, sampling settings and requests at a time as the run it replaces and a server from the same launcher (the earlier run's own server flags were not recorded); the earlier run stays in history; an earlier attempt of this same run stopped part-way (1,210 recorded pairs and 64 further requests possibly answered but not recorded) and is kept apart, not merged; this attempt started again from the first pair, so its outcomes are this attempt's recorded outputs, not guaranteed to be the model's first outputs on those pairs",
   "gpu_hours": 0.3654148079916184,
   "gpu_cost_usd": null
  },
  {
   "name": "Clef-Flash",
   "display_name": "Clef-Flash",
   "tag": "",
   "model_key": "clef-flash",
   "maker": "Cloudflare",
   "kind": "decision",
   "repo": "Cloudflare/clef-flash",
   "commit": "fde727a287004204b7518dcc983fe64379776712",
   "params_total": 9.0,
   "params_active": 9.0,
   "quantization": "BF16",
   "score_0_100": 69.76800386660223,
   "raw": {
    "metric": "accuracy",
    "value": 0.7984533591106815,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.05931271145480908,
   "improvement_points": null,
   "runs": [
    "grading:site-clef-flash-v1"
   ],
   "notes": "Cloudflare/clef-flash@fde727a2, the server its Logic & Math runs use, BF16, on one RTX PRO 6000 (ours), self-hosted; server probabilities are rounded to four decimal places and retained without renormalization; ECE uses the reported maximum probability; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.08946663268202902,
   "gpu_cost_usd": null
  },
  {
   "name": "Gemma 4 12B Unified",
   "display_name": "Gemma 4 12B Unified",
   "tag": "thinking on · chat",
   "model_key": "gemma-4-12b-think",
   "maker": "Google",
   "kind": "chat",
   "repo": "google/gemma-4-12B-it",
   "commit": "43c9d014b231474fa7212dbb48a762566c2b255d",
   "params_total": 11.95,
   "params_active": 11.95,
   "quantization": "BF16",
   "score_0_100": 67.66553890768488,
   "raw": {
    "metric": "accuracy",
    "value": 0.7844369260512325,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "grading:chat-gemma-4-12b-think-r2-v1"
   ],
   "notes": "google/gemma-4-12B-it@43c9d014, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time (latency not comparable across different concurrency); thinking on; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 296 of 2069 replies (14%) had no answer marker and count wrong (295 of them cut off at the 32,768-token limit); a fresh run (14-15 Nov) with every reply kept, with the same model, sampling settings and requests at a time as the run it replaces and a server from the same launcher (the earlier run's own server flags were not recorded); the earlier run stays in history",
   "gpu_hours": 2.486874496019143,
   "gpu_cost_usd": null
  },
  {
   "name": "MiniCPM5-2B",
   "display_name": "MiniCPM5-2B",
   "tag": "thinking off · chat",
   "model_key": "minicpm5-2b-nothink",
   "maker": "OpenBMB",
   "kind": "chat",
   "repo": "openbmb/MiniCPM5-2B",
   "commit": "42433e90dad95123ab87729fd7011e7110629556",
   "params_total": 2.52,
   "params_active": 2.52,
   "quantization": "BF16",
   "score_0_100": 66.988883518608,
   "raw": {
    "metric": "accuracy",
    "value": 0.7799258901240534,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "grading:chat-minicpm5-2b-nothink-r2-v1",
    "grading:chat-minicpm5-2b-nothink-seed1-r2-v1",
    "grading:chat-minicpm5-2b-nothink-seed2-r2-v1"
   ],
   "notes": "openbmb/MiniCPM5-2B@42433e90, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time from the client (latency not comparable across different concurrency); mean of 3 full runs, seeds 0, 1, 2 (min-max across seeds, not a confidence interval); answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; replies without an answer marker count wrong: mean rate 0.24% (per seed in seed_runs; cut off at the token limit per seed: 1, 0, 0); seed 0 is the published single run (14 Nov, every reply kept); seeds 1 and 2 are new runs (15 Nov, every reply kept) with the same model, sampling settings, requests at a time and server launcher, only the seed changed, their server settings recorded; gpu_hours is the mean of the three runs (GPU seconds per seed: 219, 76, 76, at the same recorded settings; the cause of the difference is not known)",
   "gpu_hours": 0.02949326808467177,
   "gpu_cost_usd": null,
   "accuracy_basis": "mean of 3 seeds",
   "seed_min": 0.7796036732721121,
   "seed_max": 0.7800869985500242,
   "seed_runs": [
    {
     "run": "grading:chat-minicpm5-2b-nothink-r2-v1",
     "seed": 0,
     "accuracy": 0.7800869985500242,
     "invalid": 6,
     "cutoff": 1,
     "wrong_answers_caught": 93
    },
    {
     "run": "grading:chat-minicpm5-2b-nothink-seed1-r2-v1",
     "seed": 1,
     "accuracy": 0.7800869985500242,
     "invalid": 5,
     "cutoff": 0,
     "wrong_answers_caught": 94
    },
    {
     "run": "grading:chat-minicpm5-2b-nothink-seed2-r2-v1",
     "seed": 2,
     "accuracy": 0.7796036732721121,
     "invalid": 4,
     "cutoff": 0,
     "wrong_answers_caught": 97
    }
   ],
   "invalid_rate_mean": 0.002416626389560174
  },
  {
   "name": "Qwen3.5-4B",
   "display_name": "Qwen3.5-4B",
   "tag": "thinking on · chat",
   "model_key": "qwen3.5-4b",
   "maker": "Alibaba",
   "kind": "chat",
   "repo": "Qwen/Qwen3.5-4B",
   "commit": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",
   "params_total": 4.66,
   "params_active": 4.66,
   "quantization": "BF16",
   "score_0_100": 66.21556307394877,
   "raw": {
    "metric": "accuracy",
    "value": 0.7747704204929918,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "grading:chat-qwen3.5-4b-r2-v1"
   ],
   "notes": "Qwen/Qwen3.5-4B@851bf6e8, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time (latency not comparable across different concurrency); thinking on; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 179 of 2069 replies (9%) had no answer marker and count wrong (0 of them cut off at the 32,768-token limit); a fresh run (14-15 Nov) with every reply kept, with the same model, sampling settings and requests at a time as the run it replaces and a server from the same launcher (the earlier run's own server flags were not recorded); the earlier run stays in history",
   "gpu_hours": 0.4258198070850944,
   "gpu_cost_usd": null
  },
  {
   "name": "StartLux-Decision-v1.1-2B-Instant",
   "display_name": "StartLux-Decision-v1.1-2B-Instant",
   "tag": "",
   "model_key": "startlux-decision-v1.1-2b",
   "maker": "StartLux",
   "kind": "decision",
   "repo": "startlux-models/StartLux-Decision-v1.1-2B-Instant",
   "commit": "f1e6cd147dda82c9ffa3e8e725d5c1f175566fb1",
   "params_total": 2.0,
   "params_active": 2.0,
   "quantization": "BF16",
   "score_0_100": 65.99806669888835,
   "raw": {
    "metric": "accuracy",
    "value": 0.7733204446592556,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.07203038917648069,
   "improvement_points": null,
   "runs": [
    "grading:site-startlux-decision-v1.1-2b-v1"
   ],
   "notes": "startlux-models/StartLux-Decision-v1.1-2B-Instant@f1e6cd14, the server its Logic & Math runs use, BF16, on one DGX Spark (ours), self-hosted; licence CC-BY-NC-4.0 (research use); one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.04491619354242643,
   "gpu_cost_usd": null
  },
  {
   "name": "GLiDE 2B",
   "display_name": "GLiDE 2B",
   "tag": "",
   "model_key": "glide-2b",
   "maker": "Fastino",
   "kind": "decision",
   "repo": "fastino/GLiDE-2B",
   "commit": "00f122c120cdeed82284106cb77d9bab5a50ad16",
   "params_total": 1.88,
   "params_active": 1.88,
   "quantization": "BF16",
   "score_0_100": 65.63557274045432,
   "raw": {
    "metric": "accuracy",
    "value": 0.7709038182696955,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.03954780247703763,
   "improvement_points": null,
   "runs": [
    "grading:site-glide-2b-v1"
   ],
   "notes": "fastino/GLiDE-2B@00f122c1, GLiDE server (python -m glide.server) in vLLM v0.30.0 image with flash-linear-attention 0.5.2 on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.024994372462938563,
   "gpu_cost_usd": null
  },
  {
   "name": "Qwen3.5-4B",
   "display_name": "Qwen3.5-4B",
   "tag": "thinking off · chat",
   "model_key": "qwen3.5-4b-nothink",
   "maker": "Alibaba",
   "kind": "chat",
   "repo": "Qwen/Qwen3.5-4B",
   "commit": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",
   "params_total": 4.66,
   "params_active": 4.66,
   "quantization": "BF16",
   "score_0_100": 64.62058965683906,
   "raw": {
    "metric": "accuracy",
    "value": 0.764137264378927,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "grading:chat-qwen3.5-4b-nothink-v1",
    "grading:chat-qwen3.5-4b-nothink-seed1-v1",
    "grading:chat-qwen3.5-4b-nothink-seed2-v1"
   ],
   "notes": "Qwen/Qwen3.5-4B@851bf6e8, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time from the client (latency not comparable across different concurrency); mean of 3 full runs, seeds 0, 1, 2 (min-max across seeds, not a confidence interval); answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; replies without an answer marker count wrong: mean rate 0.97% (per seed in seed_runs; cut off at the token limit per seed: 0, 1, 0); seed 0 is the published single run (every reply kept); seeds 1 and 2 are new runs (16 Nov, every reply kept) with the same model, sampling settings, requests at a time and server launcher, only the seed changed, their server settings recorded; gpu_hours is the mean of the three runs",
   "gpu_hours": 0.0852996435253031,
   "gpu_cost_usd": null,
   "accuracy_basis": "mean of 3 seeds",
   "seed_min": 0.7631706138231029,
   "seed_max": 0.7655872402126631,
   "seed_runs": [
    {
     "run": "grading:chat-qwen3.5-4b-nothink-v1",
     "seed": 0,
     "accuracy": 0.763653939101015,
     "invalid": 20,
     "cutoff": 0,
     "wrong_answers_caught": 196
    },
    {
     "run": "grading:chat-qwen3.5-4b-nothink-seed1-v1",
     "seed": 1,
     "accuracy": 0.7631706138231029,
     "invalid": 25,
     "cutoff": 1,
     "wrong_answers_caught": 202
    },
    {
     "run": "grading:chat-qwen3.5-4b-nothink-seed2-v1",
     "seed": 2,
     "accuracy": 0.7655872402126631,
     "invalid": 15,
     "cutoff": 0,
     "wrong_answers_caught": 206
    }
   ],
   "invalid_rate_mean": 0.009666505558240696
  },
  {
   "name": "Metask-Jev-4B",
   "display_name": "Metask-Jev-4B",
   "tag": "",
   "model_key": "metask-jev-4b",
   "maker": "Metask Lab",
   "kind": "decision",
   "repo": "wayfind/metask-jev-4b-policy-mix",
   "commit": "ea20fe85b28733b1522721dec119bec50947a869",
   "params_total": 4.54,
   "params_active": 4.54,
   "quantization": "BF16",
   "score_0_100": 61.720637989366836,
   "raw": {
    "metric": "accuracy",
    "value": 0.7448042532624456,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.06794952198236447,
   "improvement_points": null,
   "runs": [
    "grading:site-metask-jev-4b-v1"
   ],
   "notes": "wayfind/metask-jev-4b-policy-mix@ea20fe85, the server its Logic & Math runs use, BF16, on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.07693864276212224,
   "gpu_cost_usd": null
  },
  {
   "name": "KnowLine-2B-Gen1",
   "display_name": "KnowLine-2B-Gen1",
   "tag": "",
   "model_key": "knowline-2b-gen1",
   "maker": "PelaAI",
   "kind": "decision",
   "repo": "PelaAI/KnowLine-2B-Gen1",
   "commit": "53245d470a4ce316e32fd0d18c24e142c697f40b",
   "params_total": 2.0,
   "params_active": 2.0,
   "quantization": "BF16",
   "score_0_100": 61.06814886418559,
   "raw": {
    "metric": "accuracy",
    "value": 0.7404543257612373,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.026506755692241563,
   "improvement_points": null,
   "runs": [
    "grading:site-knowline-2b-gen1-v1"
   ],
   "notes": "PelaAI/KnowLine-2B-Gen1@53245d47, the server its Logic & Math runs use, BF16, on one DGX Spark (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.07461765998593127,
   "gpu_cost_usd": null
  },
  {
   "name": "GLiDE 0.8B",
   "display_name": "GLiDE 0.8B",
   "tag": "",
   "model_key": "glide-0.8b",
   "maker": "Fastino",
   "kind": "decision",
   "repo": "fastino/GLiDE-0.8B",
   "commit": "9eacdeec79fa48ec52a8d6f54b331376b4bbfaf0",
   "params_total": 0.75,
   "params_active": 0.75,
   "quantization": "BF16",
   "score_0_100": 60.77815369743836,
   "raw": {
    "metric": "accuracy",
    "value": 0.7385210246495891,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.03848586578633559,
   "improvement_points": null,
   "runs": [
    "grading:site-glide-0.8b-v1"
   ],
   "notes": "fastino/GLiDE-0.8B@9eacdeec, the server its Logic & Math runs use, BF16, on one DGX Spark (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.036354799436418,
   "gpu_cost_usd": null
  },
  {
   "name": "Strands Decider 2B",
   "display_name": "Strands Decider 2B",
   "tag": "",
   "model_key": "strands-decider-2b",
   "maker": "Strands Agents",
   "kind": "decision",
   "repo": "StrandsAgents/strands-decider-2B-hobson-v21",
   "commit": "2b52a6235c1b8306bbfa30b00b9d4b74b63a39f5",
   "params_total": 2.0,
   "params_active": 2.0,
   "quantization": "BF16",
   "score_0_100": 60.34316094731754,
   "raw": {
    "metric": "accuracy",
    "value": 0.735621072982117,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.04984509424842916,
   "improvement_points": null,
   "runs": [
    "grading:site-strands-decider-2b-v1"
   ],
   "notes": "StrandsAgents/strands-decider-2B-hobson-v21@2b52a623 (a LoRA and head on Qwen3.5-2B-Base), the server its Logic & Math runs use, BF16, on one RTX PRO 6000 (ours), self-hosted; server probabilities are rounded to four decimal places and retained without renormalization; ECE uses the reported maximum probability; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.02474473185662914,
   "gpu_cost_usd": null
  },
  {
   "name": "LFM2.5-8B-A1B",
   "display_name": "LFM2.5-8B-A1B",
   "tag": "thinking on (no off mode) · chat",
   "model_key": "lfm2.5-8b-a1b-think",
   "maker": "Liquid AI",
   "kind": "chat",
   "repo": "LiquidAI/LFM2.5-8B-A1B",
   "commit": "dd83f5cb02aa2938b66bba9a9998d38d0517c1f3",
   "params_total": 8.3,
   "params_active": 1.5,
   "quantization": "BF16",
   "score_0_100": 57.733204446592545,
   "raw": {
    "metric": "accuracy",
    "value": 0.7182213629772837,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "grading:chat-lfm2.5-8b-a1b-think-v1"
   ],
   "notes": "LiquidAI/LFM2.5-8B-A1B@dd83f5cb, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 30 requests at a time (latency not comparable across different concurrency); thinking on; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 0 of 2069 replies (0%) had no answer marker and count wrong; every reply kept; an earlier attempt of this same run stopped part-way (45 recorded pairs and an unknown number of further requests possibly answered but not recorded) and is kept apart, not merged; this attempt started again from the first pair, so its outcomes are this attempt's recorded outputs, not guaranteed to be the model's first outputs on those pairs",
   "gpu_hours": 0.2022492439606659,
   "gpu_cost_usd": null
  },
  {
   "name": "Gemma 4 E2B",
   "display_name": "Gemma 4 E2B",
   "tag": "thinking off · sampled (temperature 1.0) · chat",
   "model_key": "gemma-4-e2b-sampled",
   "maker": "Google",
   "kind": "chat",
   "repo": "google/gemma-4-E2B-it",
   "commit": "4adf15748de9e1a0097b6274cee9c15836c41c37",
   "params_total": 5.1,
   "params_active": 5.1,
   "quantization": "BF16",
   "score_0_100": 55.171580473658764,
   "raw": {
    "metric": "accuracy",
    "value": 0.7011438698243918,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "grading:chat-gemma-4-e2b-sampled-seed0-v1",
    "grading:chat-gemma-4-e2b-sampled-seed1-v1",
    "grading:chat-gemma-4-e2b-sampled-seed2-v1"
   ],
   "notes": "google/gemma-4-E2B-it@4adf1574, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time from the client (latency not comparable across different concurrency); mean of 3 full runs, seeds 0, 1, 2 (min-max across seeds, not a confidence interval); answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; replies without an answer marker count wrong: mean rate 0.03% (per seed in seed_runs; cut off at the token limit per seed: 0, 0, 0); three new runs (16 Nov, every reply kept), seeds 0, 1, 2 at Gemma's own sampling (its generation_config.json: temperature 1.0, top_p 0.95, top_k 64; Ümit #10554), with the weights, template and token limit of the temperature-0 row, which stays as its own row; their server settings recorded; gpu_hours is the mean of the three runs",
   "gpu_hours": 0.018991241379863,
   "gpu_cost_usd": null,
   "accuracy_basis": "mean of 3 seeds",
   "seed_min": 0.6969550507491542,
   "seed_max": 0.7056549057515709,
   "seed_runs": [
    {
     "run": "grading:chat-gemma-4-e2b-sampled-seed0-v1",
     "seed": 0,
     "accuracy": 0.6969550507491542,
     "invalid": 1,
     "cutoff": 0,
     "wrong_answers_caught": 22
    },
    {
     "run": "grading:chat-gemma-4-e2b-sampled-seed1-v1",
     "seed": 1,
     "accuracy": 0.7056549057515709,
     "invalid": 0,
     "cutoff": 0,
     "wrong_answers_caught": 29
    },
    {
     "run": "grading:chat-gemma-4-e2b-sampled-seed2-v1",
     "seed": 2,
     "accuracy": 0.7008216529724505,
     "invalid": 1,
     "cutoff": 0,
     "wrong_answers_caught": 25
    }
   ],
   "invalid_rate_mean": 0.0003222168519413565
  },
  {
   "name": "Gemma 4 E2B",
   "display_name": "Gemma 4 E2B",
   "tag": "thinking off · chat",
   "model_key": "gemma-4-e2b",
   "maker": "Google",
   "kind": "chat",
   "repo": "google/gemma-4-E2B-it",
   "commit": "4adf15748de9e1a0097b6274cee9c15836c41c37",
   "params_total": 5.1,
   "params_active": 5.1,
   "quantization": "BF16",
   "score_0_100": 54.76075398743354,
   "raw": {
    "metric": "accuracy",
    "value": 0.6984050265828903,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "grading:chat-gemma-4-e2b-r2-v1"
   ],
   "notes": "google/gemma-4-E2B-it@4adf1574, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time (latency not comparable across different concurrency); thinking off; answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; 0 of 2069 replies (0%) had no answer marker and count wrong; a fresh run (14-15 Nov) with every reply kept, with the same model, sampling settings and requests at a time as the run it replaces and a server from the same launcher (the earlier run's own server flags were not recorded); the earlier run stays in history",
   "gpu_hours": 0.010497344808671188,
   "gpu_cost_usd": null
  },
  {
   "name": "Qwen3.5-2B",
   "display_name": "Qwen3.5-2B",
   "tag": "thinking off · chat",
   "model_key": "qwen3.5-2b-nothink",
   "maker": "Alibaba",
   "kind": "chat",
   "repo": "Qwen/Qwen3.5-2B",
   "commit": "15852e8c16360a2fea060d615a32b45270f8a8fc",
   "params_total": 2.0,
   "params_active": 2.0,
   "quantization": "BF16",
   "score_0_100": 50.24166263895601,
   "raw": {
    "metric": "accuracy",
    "value": 0.6682777509263734,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "grading:chat-qwen3.5-2b-nothink-v1",
    "grading:chat-qwen3.5-2b-nothink-seed1-v1",
    "grading:chat-qwen3.5-2b-nothink-seed2-v1"
   ],
   "notes": "Qwen/Qwen3.5-2B@15852e8c, BF16, v0.30.0 vLLM on one RTX PRO 6000 (ours), self-hosted; one question per request, 64 requests at a time from the client (latency not comparable across different concurrency); mean of 3 full runs, seeds 0, 1, 2 (min-max across seeds, not a confidence interval); answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; replies without an answer marker count wrong: mean rate 2% (per seed in seed_runs; cut off at the token limit per seed: 1, 0, 0); seed 0 is the published single run (every reply kept); seeds 1 and 2 are new runs (16 Nov, every reply kept) with the same model, sampling settings, requests at a time and server launcher, only the seed changed, their server settings recorded; gpu_hours is the mean of the three runs",
   "gpu_hours": 0.052665647040142136,
   "gpu_cost_usd": null,
   "accuracy_basis": "mean of 3 seeds",
   "seed_min": 0.6621556307394877,
   "seed_max": 0.6718221362977284,
   "seed_runs": [
    {
     "run": "grading:chat-qwen3.5-2b-nothink-v1",
     "seed": 0,
     "accuracy": 0.6621556307394877,
     "invalid": 37,
     "cutoff": 1,
     "wrong_answers_caught": 108
    },
    {
     "run": "grading:chat-qwen3.5-2b-nothink-seed1-v1",
     "seed": 1,
     "accuracy": 0.6708554857419043,
     "invalid": 45,
     "cutoff": 0,
     "wrong_answers_caught": 116
    },
    {
     "run": "grading:chat-qwen3.5-2b-nothink-seed2-v1",
     "seed": 2,
     "accuracy": 0.6718221362977284,
     "invalid": 43,
     "cutoff": 0,
     "wrong_answers_caught": 116
    }
   ],
   "invalid_rate_mean": 0.020138553246334782
  },
  {
   "name": "OneJev1.6-0.8B",
   "display_name": "OneJev1.6-0.8B",
   "tag": "",
   "model_key": "onejev1.6-0.8b",
   "maker": "OmniJev",
   "kind": "decision",
   "repo": "OmniJev/OneJev1.6-0.8B",
   "commit": "bcd88a382f314b26cb7b1b7f369c1ed3502ee7a0",
   "params_total": 0.8,
   "params_active": 0.8,
   "quantization": "BF16",
   "score_0_100": 50.0483325277912,
   "raw": {
    "metric": "accuracy",
    "value": 0.6669888835186081,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.040721618656355756,
   "improvement_points": null,
   "runs": [
    "grading:site-onejev1.6-0.8b-v1"
   ],
   "notes": "OmniJev/OneJev1.6-0.8B@bcd88a38, the server its Logic & Math runs use, BF16, on one DGX Spark (ours), self-hosted; server probabilities are rounded to six decimal places and retained without renormalization; ECE uses the reported maximum probability; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.024637469203680373,
   "gpu_cost_usd": null
  },
  {
   "name": "d1-2B",
   "display_name": "d1-2B",
   "tag": "",
   "model_key": "liquid-d1-2b",
   "maker": "Liquid AI",
   "kind": "decision",
   "repo": "LiquidAI/d1-2B",
   "commit": "10e7671d44e4b37700020b1eed3f594c5f2af0b4",
   "params_total": 1.92,
   "params_active": 1.92,
   "quantization": "BF16",
   "score_0_100": 49.613339777670376,
   "raw": {
    "metric": "accuracy",
    "value": 0.6640889318511358,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.07187269491787361,
   "improvement_points": null,
   "runs": [
    "grading:site-liquid-d1-2b-v1"
   ],
   "notes": "LiquidAI/d1-2B@10e7671d, d1_server (Liquid's model code) on one RTX PRO 6000 (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.014600491343951257,
   "gpu_cost_usd": null
  },
  {
   "name": "StartLux-Decision-v1.1-0.8B-Instant",
   "display_name": "StartLux-Decision-v1.1-0.8B-Instant",
   "tag": "",
   "model_key": "startlux-decision-v1.1-0.8b-instant",
   "maker": "StartLux",
   "kind": "decision",
   "repo": "startlux-models/StartLux-Decision-v1.1-0.8B-Instant",
   "commit": "6496ae641cc71b149ce9dad69bd97f2a44e7bf11",
   "params_total": 0.8,
   "params_active": 0.8,
   "quantization": "BF16",
   "score_0_100": 48.88835186080232,
   "raw": {
    "metric": "accuracy",
    "value": 0.6592556790720154,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.056035763218873734,
   "improvement_points": null,
   "runs": [
    "grading:site-startlux-decision-v1.1-0.8b-instant-v1"
   ],
   "notes": "startlux-models/StartLux-Decision-v1.1-0.8B-Instant@6496ae64, the server its Logic & Math runs use, BF16, on one DGX Spark (ours), self-hosted; licence CC-BY-NC-4.0 (research use); one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.030659771969914984,
   "gpu_cost_usd": null
  },
  {
   "name": "GLiDE 0.4B",
   "display_name": "GLiDE 0.4B",
   "tag": "",
   "model_key": "glide-0.4b",
   "maker": "Fastino",
   "kind": "decision",
   "repo": "fastino/GLiDE-0.4B",
   "commit": "6bda7ab91aeb36dcad6d27828a8871e548c7fc71",
   "params_total": 0.4,
   "params_active": 0.4,
   "quantization": "BF16",
   "score_0_100": 44.393426776220394,
   "raw": {
    "metric": "accuracy",
    "value": 0.6292895118414693,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.07677301660636211,
   "improvement_points": null,
   "runs": [
    "grading:site-glide-0.4b-v1"
   ],
   "notes": "fastino/GLiDE-0.4B@6bda7ab9, the server its Logic & Math runs use, BF16, on one DGX Spark (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.023397810828577028,
   "gpu_cost_usd": null
  },
  {
   "name": "GLiClass v4 Edge",
   "display_name": "GLiClass v4 Edge",
   "tag": "",
   "model_key": "gliclass-v4-edge",
   "maker": "Knowledgator",
   "kind": "decision",
   "repo": "knowledgator/gliclass-v4-edge",
   "commit": "885af1af8a07edf4ed34df83ca0d995ed8201283",
   "params_total": 0.4,
   "params_active": 0.4,
   "quantization": "BF16",
   "score_0_100": 44.32092798453359,
   "raw": {
    "metric": "accuracy",
    "value": 0.6288061865635572,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.09842112173409612,
   "improvement_points": null,
   "runs": [
    "grading:site-gliclass-v4-edge-v1"
   ],
   "notes": "knowledgator/gliclass-v4-edge@885af1af, the server its Logic & Math runs use, BF16, on one DGX Spark (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.07387427885198607,
   "gpu_cost_usd": null
  },
  {
   "name": "Decider 1",
   "display_name": "Decider 1",
   "tag": "",
   "model_key": "meragpt-decider-1",
   "maker": "meraGPT",
   "kind": "decision",
   "repo": null,
   "commit": null,
   "params_total": null,
   "params_active": null,
   "quantization": null,
   "score_0_100": 43.30594490091833,
   "raw": {
    "metric": "accuracy",
    "value": 0.6220396326727888,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.059314403093281794,
   "improvement_points": null,
   "runs": [
    "grading:site-meragpt-decider-1-v1"
   ],
   "notes": "meraGPT decisions API (meragpt.com, model decider-1); one question per request, 8 requests at a time (latency not comparable across different concurrency)",
   "gpu_hours": null,
   "gpu_cost_usd": null
  },
  {
   "name": "Raon-1 nano",
   "display_name": "Raon-1 nano",
   "tag": "thinking off · chat",
   "model_key": "raon-1-nano-nothink",
   "maker": "KRAFTON",
   "kind": "chat",
   "repo": "KRAFTON/Raon-1-nano",
   "commit": "fcf2d1d55e84e2f9fbe7b5d724dcfc4d861cfcee",
   "params_total": 3.1,
   "params_active": 1.5,
   "quantization": "BF16",
   "score_0_100": 40.647655872402126,
   "raw": {
    "metric": "accuracy",
    "value": 0.6043177058160142,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": null,
   "improvement_points": null,
   "runs": [
    "grading:chat-raon-1-nano-nothink-v1",
    "grading:chat-raon-1-nano-nothink-seed1-v1",
    "grading:chat-raon-1-nano-nothink-seed2-v1"
   ],
   "notes": "KRAFTON/Raon-1-nano@fcf2d1d5, BF16, KRAFTON vLLM fork krafton-ai/vllm@13a5dd87 (not a vLLM release) on one RTX PRO 6000 (ours), self-hosted; one question per request, 30 requests at a time from the client (latency not comparable across different concurrency); mean of 3 full runs, seeds 0, 1, 2 (min-max across seeds, not a confidence interval); answer rule v2: the label is the last marker 'ANSWER: X' anywhere in the visible reply, where the word ANSWER may be in any letter case and X must be an uppercase A, B or C; the reasoning text is never read; v2 replaced rule v1 (the word in uppercase only) on 15 Nov after v1 was found to miss replies written 'Answer: X', and applies to every chat row; replies without an answer marker count wrong: mean rate 0.03% (per seed in seed_runs; cut off at the token limit per seed: 0, 1, 0); seed 0 is the published single run (every reply kept); seeds 1 and 2 are new runs (16 Nov, every reply kept) with the same model, sampling settings, requests at a time and server launcher, only the seed changed, their server settings recorded; gpu_hours is the mean of the three runs",
   "gpu_hours": 0.11989139371652192,
   "gpu_cost_usd": null,
   "accuracy_basis": "mean of 3 seeds",
   "seed_min": 0.6036732721121315,
   "seed_max": 0.6051232479458676,
   "seed_runs": [
    {
     "run": "grading:chat-raon-1-nano-nothink-v1",
     "seed": 0,
     "accuracy": 0.6051232479458676,
     "invalid": 0,
     "cutoff": 0,
     "wrong_answers_caught": 65
    },
    {
     "run": "grading:chat-raon-1-nano-nothink-seed1-v1",
     "seed": 1,
     "accuracy": 0.6041565973900435,
     "invalid": 1,
     "cutoff": 1,
     "wrong_answers_caught": 68
    },
    {
     "run": "grading:chat-raon-1-nano-nothink-seed2-v1",
     "seed": 2,
     "accuracy": 0.6036732721121315,
     "invalid": 1,
     "cutoff": 0,
     "wrong_answers_caught": 63
    }
   ],
   "invalid_rate_mean": 0.0003222168519413565
  },
  {
   "name": "KnowLine-0.8B-Gen1",
   "display_name": "KnowLine-0.8B-Gen1",
   "tag": "",
   "model_key": "knowline-0.8b-gen1",
   "maker": "PelaAI",
   "kind": "decision",
   "repo": "PelaAI/KnowLine-0.8B-Gen1",
   "commit": "6b27831f7b783c03278bd946d4d8f4f7a98a3798",
   "params_total": 0.8,
   "params_active": 0.8,
   "quantization": "BF16",
   "score_0_100": 38.52102464958917,
   "raw": {
    "metric": "accuracy",
    "value": 0.5901401643305945,
    "questions": 2069,
    "options": 3
   },
   "calibration_error": 0.14642663407717668,
   "improvement_points": null,
   "runs": [
    "grading:site-knowline-0.8b-gen1-v1"
   ],
   "notes": "PelaAI/KnowLine-0.8B-Gen1@6b27831f, the server its Logic & Math runs use, BF16, on one DGX Spark (ours), self-hosted; one question per request, 1 request at a time (latency not comparable across different concurrency)",
   "gpu_hours": 0.04669099587283346,
   "gpu_cost_usd": null
  }
 ]
}
