{
  "report_ids": [
    "f6a742942de067fc3de4686e688c564e80be956f0bbeee5b08c5f96d12e160af",
    "49054f114d4aa8ab00b0c970a466e93e09bdb8a8a2b13e094cb3f89fa4ccb6f5",
    "6438dc0d15728a182dc3216c2f94299e8263235b2e728a125ff9f8cadbdc76ed",
    "53112cc9e51a3c84619a497052868f2be47fd3bfcd44b4eb17af43f0f08311a9",
    "27a84c9ac606f29120025ba5146dab10d8f79ef462e0349180345e8f555e1039",
    "14a1670c43f9b70edb6d260fd3652c7187b98c97fd041848fce7f700d36730ff",
    "21f3ef13e659ef514e71016f70155fa31e593844002b71343fc4d06cde56b3fe",
    "5a64e9a304ce5c22dfd4735c60096c3fe6de65503565fd9c137b08a2bde90f1e",
    "10f52280f44dd977a0733509d0804f1816b0e0644c25d892d70f1f66e9802318",
    "6e10091742dfb5f0288682ffb0410d23bff860b2d03558c320630b957452baf6",
    "d2f720cf1b957dd753d1e58d2e828032b15b14a9ab92d897881a20200f6195e1",
    "24f1c17bca4f3b38537e72c6986fabd7b26ff4299c7e541839e3202593272272",
    "96415472354b896e265793e2365fc5b2bebc5594968e98f2c325bb3c39f43339",
    "d22fa30c9004d04be7e72181e46b3f7fdd3ec9120580253f3f1e57f22d72c827",
    "79f5324a24db4cc6683f5865e00eb7d8331fe137d891fbdf13dfa98cb58be2d3",
    "511dae62ca5252589cc84079f21e12e5a4e9da5859b8010ed9dba2db800e6cc2",
    "87114d424d910acc4d84092ad931c0475cb0d5065dd1225e549e670a2d5b032c",
    "e03a41c8c729e138a1fb4130837439a8db729ee21829c5ba0d89b80306493c30",
    "9918f896f7234fe98b21795fb4937fb29b7dfe49b233ddc1bdda4c5881fb2c8e",
    "33b2fc529fb2de88831d43e81b9afd563c90d7d8dbded7d334a2ac02081daff4",
    "604dc3902cff19025cbfe96022094ed0376406a03797afe689cb8227ffe6786c",
    "f4b2e4a175cbb720ef525cacf7c1ce04ae09083a794457d2af61fbcf1f78e024",
    "a92b0fc63f5e4d1d3c6ad852aca000c3a793ff08d84ee076b0ab1367a5930506"
  ],
  "reports": [
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 0.75,
          "passed": 18,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 0.375,
          "passed": 9,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 0.08333333333333333,
          "passed": 2,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 0.0,
          "passed": 0,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.125,
          "passed": 3,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 0.4166666666666667,
          "passed": 10,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "tiny",
        "class_label": "Open \u00b7 1.7B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/LiquidAI/LFM3-1.7B/blob/0dc4268b561499c648c7ba7c9466bf3d8d01517a/README.md",
        "maker": "Liquid AI",
        "params_active": 1.73,
        "params_total": 1.73,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "bad_format": 18,
        "extra_call": 6,
        "gave_up_or_loop": 4,
        "wrong_args": 73,
        "wrong_tool": 1
      },
      "gpu": {
        "card": "bariton GPU 1 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 22.5,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 32
      },
      "latency_seconds": {
        "mean": 2.145355525881996,
        "total": 308.93119572700743
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.2916666666666667,
        "passed": 42,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_tool",
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "0dc4268b561499c648c7ba7c9466bf3d8d01517a",
        "engine_tool_parser": "qwen3_xml",
        "exact_model_id": "LiquidAI/LFM3-1.7B",
        "native_documented_call_format": "openai tool_calls via engine tool parser qwen3_xml",
        "plan_sha256": "ba105e1f98f38e5f7032392f3ec701cae4ab2ad9d3b976cf14d0b0a4425a81eb",
        "provider": "local vLLM served by @logic on bariton (#9528 standard set; #10380 hard set, identical argv)",
        "requested_name": "LFM3-1.7B",
        "revision": "0dc4268b561499c648c7ba7c9466bf3d8d01517a",
        "thinking_mode": "off"
      },
      "tiers": {
        "all": 3,
        "original": 2,
        "renamed": 2
      },
      "total_tokens": 1340110,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.3333333333333333,
          "passed": 24,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.25,
          "passed": 18,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.2222222222222222,
          "passed": 8,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.3148148148148148,
          "passed": 34,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 0.5833333333333334,
          "passed": 14,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 0.75,
          "passed": 18,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 0.5,
          "passed": 12,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.25,
          "passed": 6,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 0.7916666666666666,
          "passed": 19,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "tiny",
        "class_label": "Open \u00b7 1.7B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/LiquidAI/LFM3-1.7B/blob/0dc4268b561499c648c7ba7c9466bf3d8d01517a/README.md",
        "maker": "Liquid AI",
        "params_active": 1.73,
        "params_total": 1.73,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "bad_format": 25,
        "extra_call": 5,
        "wrong_args": 21
      },
      "gpu": {
        "card": "bariton GPU 1 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 51.3,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 32
      },
      "latency_seconds": {
        "mean": 4.834793886999908,
        "total": 696.2103197279866
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.6458333333333334,
        "passed": 93,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "0dc4268b561499c648c7ba7c9466bf3d8d01517a",
        "engine_tool_parser": "qwen3_xml",
        "exact_model_id": "LiquidAI/LFM3-1.7B",
        "native_documented_call_format": "openai tool_calls via engine tool parser qwen3_xml",
        "plan_sha256": "ba105e1f98f38e5f7032392f3ec701cae4ab2ad9d3b976cf14d0b0a4425a81eb",
        "provider": "local vLLM served by @logic on bariton (#9528 standard set; #10380 hard set, identical argv)",
        "requested_name": "LFM3-1.7B",
        "revision": "0dc4268b561499c648c7ba7c9466bf3d8d01517a",
        "thinking_mode": "on"
      },
      "tiers": {
        "all": 2,
        "original": 2,
        "renamed": 1
      },
      "total_tokens": 1790981,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.6527777777777778,
          "passed": 47,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.6388888888888888,
          "passed": 46,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.8055555555555556,
          "passed": 29,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.5925925925925926,
          "passed": 64,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 0.875,
          "passed": 21,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 0.625,
          "passed": 15,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 0.7916666666666666,
          "passed": 19,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.2916666666666667,
          "passed": 7,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 0.8333333333333334,
          "passed": 20,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "tiny",
        "class_label": "Open \u00b7 5.1B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/google/gemma-4-E2B-it/blob/4adf15748de9e1a0097b6274cee9c15836c41c37/README.md",
        "maker": "Google",
        "params_active": 5.1,
        "params_total": 5.1,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "bad_format": 11,
        "wrong_args": 27
      },
      "gpu": {
        "card": "bariton GPU 1 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 106.1,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 16
      },
      "latency_seconds": {
        "mean": 10.974158113110953,
        "total": 1580.278768287977
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.7361111111111112,
        "passed": 106,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "4adf15748de9e1a0097b6274cee9c15836c41c37",
        "engine_tool_parser": "gemma4",
        "exact_model_id": "google/gemma-4-E2B-it",
        "native_documented_call_format": "openai tool_calls via engine tool parser gemma4",
        "plan_sha256": "34abd41fbe98de8b1b913669618046fa603e82524b03b347b38e990454abdf75",
        "provider": "local vLLM served by @logic on bariton (#9534 standard set; #10411 hard set, identical argv)",
        "requested_name": "Gemma 4 E2B",
        "revision": "4adf15748de9e1a0097b6274cee9c15836c41c37",
        "thinking_mode": "on"
      },
      "tiers": {
        "all": 1,
        "original": 1,
        "renamed": 1
      },
      "total_tokens": 2048615,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.7083333333333334,
          "passed": 51,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.7638888888888888,
          "passed": 55,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.8611111111111112,
          "passed": 31,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.6944444444444444,
          "passed": 75,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 0.6666666666666666,
          "passed": 16,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 0.7083333333333334,
          "passed": 17,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 0.16666666666666666,
          "passed": 4,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 0.08333333333333333,
          "passed": 2,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.041666666666666664,
          "passed": 1,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 0.08333333333333333,
          "passed": 2,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "tiny",
        "class_label": "Open \u00b7 2.5B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/42433e90dad95123ab87729fd7011e7110629556/README.md",
        "maker": "OpenBMB",
        "params_active": 2.52,
        "params_total": 2.52,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "bad_format": 18,
        "extra_call": 7,
        "gave_up_or_loop": 6,
        "wrong_args": 71
      },
      "gpu": {
        "card": "bariton GPU 1 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 23.8,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 16
      },
      "latency_seconds": {
        "mean": 2.3525183529097453,
        "total": 338.7626428190033
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.2916666666666667,
        "passed": 42,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-branching-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "42433e90dad95123ab87729fd7011e7110629556",
        "engine_tool_parser": "minicpm5",
        "exact_model_id": "openbmb/MiniCPM5-2B",
        "native_documented_call_format": "openai tool_calls via engine tool parser minicpm5",
        "plan_sha256": "6a18df7e807182b5f66cfc249f366a35d8a395000ad36206d1069db1dbff5b60",
        "provider": "local vLLM served by @logic on bariton (#9641 standard set; #10441 hard set, identical argv)",
        "requested_name": "MiniCPM5-2B",
        "revision": "42433e90dad95123ab87729fd7011e7110629556",
        "thinking_mode": "off"
      },
      "tiers": {
        "all": 3,
        "original": 2,
        "renamed": 3
      },
      "total_tokens": 1586690,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.3611111111111111,
          "passed": 26,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.2222222222222222,
          "passed": 16,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.16666666666666666,
          "passed": 6,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.3333333333333333,
          "passed": 36,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 0.9583333333333334,
          "passed": 23,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 0.7083333333333334,
          "passed": 17,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 0.875,
          "passed": 21,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.5416666666666666,
          "passed": 13,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 0.9166666666666666,
          "passed": 22,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "tiny",
        "class_label": "Open \u00b7 2.5B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/openbmb/MiniCPM5-2B/blob/42433e90dad95123ab87729fd7011e7110629556/README.md",
        "maker": "OpenBMB",
        "params_active": 2.52,
        "params_total": 2.52,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "bad_format": 6,
        "wrong_args": 18
      },
      "gpu": {
        "card": "bariton GPU 1 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 92.1,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 16
      },
      "latency_seconds": {
        "mean": 8.33764466490259,
        "total": 1200.6208317459732
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.8333333333333334,
        "passed": 120,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "42433e90dad95123ab87729fd7011e7110629556",
        "engine_tool_parser": "minicpm5",
        "exact_model_id": "openbmb/MiniCPM5-2B",
        "native_documented_call_format": "openai tool_calls via engine tool parser minicpm5",
        "plan_sha256": "6a18df7e807182b5f66cfc249f366a35d8a395000ad36206d1069db1dbff5b60",
        "provider": "local vLLM served by @logic on bariton (#9641 standard set; #10441 hard set, identical argv)",
        "requested_name": "MiniCPM5-2B",
        "revision": "42433e90dad95123ab87729fd7011e7110629556",
        "thinking_mode": "on"
      },
      "tiers": {
        "all": 1,
        "original": 1,
        "renamed": 1
      },
      "total_tokens": 2288473,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.8333333333333334,
          "passed": 60,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.8333333333333334,
          "passed": 60,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.8055555555555556,
          "passed": 29,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.8425925925925926,
          "passed": 91,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 0.8333333333333334,
          "passed": 20,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 0.875,
          "passed": 21,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 0.2916666666666667,
          "passed": 7,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 0.75,
          "passed": 18,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.08333333333333333,
          "passed": 2,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 0.625,
          "passed": 15,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "tiny",
        "class_label": "Open \u00b7 2.0B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
        "maker": "Alibaba",
        "params_active": 2.0,
        "params_total": 2.0,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "extra_call": 3,
        "gave_up_or_loop": 8,
        "made_up_result": 1,
        "wrong_args": 47,
        "wrong_tool": 2
      },
      "gpu": {
        "card": "bariton GPU 1 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 65.8,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 16
      },
      "latency_seconds": {
        "mean": 5.865983070986001,
        "total": 844.7015622219842
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.5763888888888888,
        "passed": 83,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-branching-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "made_up_result",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
        "engine_tool_parser": "qwen3_coder",
        "exact_model_id": "Qwen/Qwen3.5-2B",
        "native_documented_call_format": "openai tool_calls via engine tool parser qwen3_coder",
        "plan_sha256": "79a15672896acf95fec5504c709d4ea9b9ce3de2ee0f5f4a9a0466ddcb7553d9",
        "provider": "local vLLM served by @logic on bariton (#9668 standard set; #10459 hard set, identical argv)",
        "requested_name": "Qwen3.5-2B",
        "revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
        "thinking_mode": "off"
      },
      "tiers": {
        "all": 2,
        "original": 2,
        "renamed": 2
      },
      "total_tokens": 2473838,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.5833333333333334,
          "passed": 42,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.5694444444444444,
          "passed": 41,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.6666666666666666,
          "passed": 24,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.5462962962962963,
          "passed": 59,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 0.8333333333333334,
          "passed": 20,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 0.4583333333333333,
          "passed": 11,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 0.75,
          "passed": 18,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.20833333333333334,
          "passed": 5,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 0.8333333333333334,
          "passed": 20,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "tiny",
        "class_label": "Open \u00b7 2.0B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/Qwen/Qwen3.5-2B/blob/15852e8c16360a2fea060d615a32b45270f8a8fc/README.md",
        "maker": "Alibaba",
        "params_active": 2.0,
        "params_total": 2.0,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "extra_call": 5,
        "wrong_args": 40,
        "wrong_tool": 1
      },
      "gpu": {
        "card": "bariton GPU 1 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 111.9,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 16
      },
      "latency_seconds": {
        "mean": 11.172464181590032,
        "total": 1608.8348421489645
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.6805555555555556,
        "passed": 98,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
        "engine_tool_parser": "qwen3_coder",
        "exact_model_id": "Qwen/Qwen3.5-2B",
        "native_documented_call_format": "openai tool_calls via engine tool parser qwen3_coder",
        "plan_sha256": "79a15672896acf95fec5504c709d4ea9b9ce3de2ee0f5f4a9a0466ddcb7553d9",
        "provider": "local vLLM served by @logic on bariton (#9668 standard set; #10459 hard set, identical argv)",
        "requested_name": "Qwen3.5-2B",
        "revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
        "thinking_mode": "on"
      },
      "tiers": {
        "all": 2,
        "original": 2,
        "renamed": 1
      },
      "total_tokens": 2603860,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.6666666666666666,
          "passed": 48,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.6944444444444444,
          "passed": 50,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.7222222222222222,
          "passed": 26,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.6666666666666666,
          "passed": 72,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 0.9166666666666666,
          "passed": 22,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 0.875,
          "passed": 21,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.625,
          "passed": 15,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 0.9583333333333334,
          "passed": 23,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "tiny",
        "class_label": "Open \u00b7 4.7B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md",
        "maker": "Alibaba",
        "params_active": 4.66,
        "params_total": 4.66,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "bad_format": 1,
        "wrong_args": 14
      },
      "gpu": {
        "card": "bariton GPU 1 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 76.4,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 16
      },
      "latency_seconds": {
        "mean": 8.047782708757003,
        "total": 1158.8807100610084
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.8958333333333334,
        "passed": 129,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",
        "engine_tool_parser": "qwen3_coder",
        "exact_model_id": "Qwen/Qwen3.5-4B",
        "native_documented_call_format": "openai tool_calls via engine tool parser qwen3_coder",
        "plan_sha256": "974f1be0b97551c4a6c332135e0fd0459f5093e1989d4d59750e76b82e38b76c",
        "provider": "local vLLM served by @logic on bariton (#9689 standard set; #10468 hard set, identical argv)",
        "requested_name": "Qwen3.5 4B",
        "revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",
        "thinking_mode": "off"
      },
      "tiers": {
        "all": 1,
        "original": 1,
        "renamed": 1
      },
      "total_tokens": 2194487,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.9444444444444444,
          "passed": 68,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.8472222222222222,
          "passed": 61,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.9444444444444444,
          "passed": 34,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.8796296296296297,
          "passed": 95,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 0.9583333333333334,
          "passed": 23,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 0.8333333333333334,
          "passed": 20,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 0.9166666666666666,
          "passed": 22,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 0.875,
          "passed": 21,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.375,
          "passed": 9,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 0.875,
          "passed": 21,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "tiny",
        "class_label": "Open \u00b7 4.7B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/Qwen/Qwen3.5-4B/blob/851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a/README.md",
        "maker": "Alibaba",
        "params_active": 4.66,
        "params_total": 4.66,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "bad_format": 3,
        "wrong_args": 25
      },
      "gpu": {
        "card": "bariton GPU 1 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 149.7,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 16
      },
      "latency_seconds": {
        "mean": 15.284565127395972,
        "total": 2200.97737834502
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.8055555555555556,
        "passed": 116,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",
        "engine_tool_parser": "qwen3_coder",
        "exact_model_id": "Qwen/Qwen3.5-4B",
        "native_documented_call_format": "openai tool_calls via engine tool parser qwen3_coder",
        "plan_sha256": "974f1be0b97551c4a6c332135e0fd0459f5093e1989d4d59750e76b82e38b76c",
        "provider": "local vLLM served by @logic on bariton (#9689 standard set; #10468 hard set, identical argv)",
        "requested_name": "Qwen3.5 4B",
        "revision": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",
        "thinking_mode": "on"
      },
      "tiers": {
        "all": 1,
        "original": 1,
        "renamed": 1
      },
      "total_tokens": 2457570,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.7777777777777778,
          "passed": 56,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.8333333333333334,
          "passed": 60,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.7777777777777778,
          "passed": 28,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.8148148148148148,
          "passed": 88,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 0.4583333333333333,
          "passed": 11,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 0.875,
          "passed": 21,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 0.041666666666666664,
          "passed": 1,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 0.041666666666666664,
          "passed": 1,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.0,
          "passed": 0,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 0.08333333333333333,
          "passed": 2,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "tiny",
        "class_label": "Open \u00b7 3.1B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/KRAFTON/Raon-1-nano/blob/fcf2d1d55e84e2f9fbe7b5d724dcfc4d861cfcee/README.md",
        "maker": "KRAFTON",
        "params_active": 3.1,
        "params_total": 3.1,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "bad_format": 1,
        "extra_call": 15,
        "gave_up_or_loop": 6,
        "wrong_args": 81,
        "wrong_tool": 5
      },
      "gpu": {
        "card": "bariton GPU 1 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 93.8,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 16
      },
      "latency_seconds": {
        "mean": 8.829320731576242,
        "total": 1271.422185346979
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.25,
        "passed": 36,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_tool",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_tool",
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_tool",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_tool",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-01-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "fcf2d1d55e84e2f9fbe7b5d724dcfc4d861cfcee",
        "engine_tool_parser": "raon_1",
        "exact_model_id": "KRAFTON/Raon-1-nano",
        "native_documented_call_format": "openai tool_calls via engine tool parser raon_1",
        "plan_sha256": "f9b4543960dd679e6545eeb2762d8bc7f0e27fccdf7f92d47da83e040d23099e",
        "provider": "local KRAFTON vLLM fork served by @logic on bariton (#9725 standard set; #10474 hard set, identical command line)",
        "requested_name": "Raon-1 nano",
        "revision": "fcf2d1d55e84e2f9fbe7b5d724dcfc4d861cfcee",
        "thinking_mode": "off"
      },
      "tiers": {
        "all": 3,
        "original": 2,
        "renamed": 3
      },
      "total_tokens": 2355910,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.2638888888888889,
          "passed": 19,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.2361111111111111,
          "passed": 17,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.19444444444444445,
          "passed": 7,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.26851851851851855,
          "passed": 29,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 0.875,
          "passed": 21,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 0.4583333333333333,
          "passed": 11,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 0.6666666666666666,
          "passed": 16,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.2916666666666667,
          "passed": 7,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 0.9583333333333334,
          "passed": 23,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "tiny",
        "class_label": "Open \u00b7 3.1B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/KRAFTON/Raon-1-nano/blob/fcf2d1d55e84e2f9fbe7b5d724dcfc4d861cfcee/README.md",
        "maker": "KRAFTON",
        "params_active": 3.1,
        "params_total": 3.1,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "extra_call": 12,
        "wrong_args": 30
      },
      "gpu": {
        "card": "bariton GPU 1 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 274.6,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 16
      },
      "latency_seconds": {
        "mean": 22.222298163368123,
        "total": 3200.0109355250097
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.7083333333333334,
        "passed": 102,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-computed-args-04-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "fcf2d1d55e84e2f9fbe7b5d724dcfc4d861cfcee",
        "engine_tool_parser": "raon_1",
        "exact_model_id": "KRAFTON/Raon-1-nano",
        "native_documented_call_format": "openai tool_calls via engine tool parser raon_1",
        "plan_sha256": "f9b4543960dd679e6545eeb2762d8bc7f0e27fccdf7f92d47da83e040d23099e",
        "provider": "local KRAFTON vLLM fork served by @logic on bariton (#9725 standard set; #10474 hard set, identical command line)",
        "requested_name": "Raon-1 nano",
        "revision": "fcf2d1d55e84e2f9fbe7b5d724dcfc4d861cfcee",
        "thinking_mode": "on"
      },
      "tiers": {
        "all": 2,
        "original": 1,
        "renamed": 1
      },
      "total_tokens": 2137047,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.7083333333333334,
          "passed": 51,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.7083333333333334,
          "passed": 51,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.8055555555555556,
          "passed": 29,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.6759259259259259,
          "passed": 73,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 0.9583333333333334,
          "passed": 23,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 0.7083333333333334,
          "passed": 17,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 0.75,
          "passed": 18,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.20833333333333334,
          "passed": 5,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "tiny",
        "class_label": "Open \u00b7 5.1B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/google/gemma-4-E2B-it/blob/4adf15748de9e1a0097b6274cee9c15836c41c37/README.md",
        "maker": "Google",
        "params_active": 5.1,
        "params_total": 5.1,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "extra_call": 1,
        "made_up_result": 5,
        "wrong_args": 27
      },
      "gpu": {
        "card": "bariton GPU 1 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 67.8,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 16
      },
      "latency_seconds": {
        "mean": 6.975783151736146,
        "total": 1004.512773850005
      },
      "note": "Non-maker template (\u00dcmit #10181): the maker's chat template plus the two-line fix proposed in https://github.com/google-deepmind/gemma/issues/829 (also added by Google to gemma-4-31B-it at commit ad62905): an empty thought '<|channel>thought\\n<channel|>' after a tool result with thinking off (template sha256 84d5a64c\u2026). Thinking is off by request, but the parser classed 58% of output tokens as reasoning (275 of 580 replies). The maker-template thinking-off setup stays 'Not qualified'.",
      "overall": {
        "attempted": 144,
        "pass_rate": 0.7708333333333334,
        "passed": 111,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "made_up_result",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "4adf15748de9e1a0097b6274cee9c15836c41c37 + evidence/gemma-off-cue-10181/chat_template-cue.jinja (non-maker: upstream gemma-4-31B-it ad62905 thinking-off cue after a tool response)",
        "engine_tool_parser": "gemma4",
        "exact_model_id": "google/gemma-4-E2B-it",
        "native_documented_call_format": "openai tool_calls via engine tool parser gemma4",
        "plan_sha256": "86c8c8e2c972b8c9d8298b7a328bde916166514d8d93c97325d0b9d03907000a",
        "provider": "local vLLM served by @logic on bariton (#10207 GPU 3 for qualification; #10260 GPU 2 for the standard run; #10423 GPU 1 for the hard set; identical argv/image/template)",
        "requested_name": "Gemma 4 E2B (tool-result cue template)",
        "revision": "4adf15748de9e1a0097b6274cee9c15836c41c37",
        "thinking_mode": "off"
      },
      "tiers": {
        "all": 1,
        "original": 1,
        "renamed": 1
      },
      "total_tokens": 1836873,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.7638888888888888,
          "passed": 55,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.7777777777777778,
          "passed": 56,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.7222222222222222,
          "passed": 26,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.7870370370370371,
          "passed": 85,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 0.6666666666666666,
          "passed": 16,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 0.625,
          "passed": 15,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 0.20833333333333334,
          "passed": 5,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 0.20833333333333334,
          "passed": 5,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.125,
          "passed": 3,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 0.5833333333333334,
          "passed": 14,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "tiny",
        "class_label": "Open \u00b7 3.0B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/ibm-granite/granite-4.2-3b/blob/4e9c275e232f10211ec88644d106a1f6a1b60e06/README.md",
        "maker": "IBM",
        "params_active": 3.0,
        "params_total": 3.0,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "extra_call": 12,
        "gave_up_or_loop": 5,
        "wrong_args": 68,
        "wrong_tool": 1
      },
      "gpu": {
        "card": "bariton GPU 2 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 267.1,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 32
      },
      "latency_seconds": {
        "mean": 12.157233652305434,
        "total": 1750.6416459319826
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.4027777777777778,
        "passed": 58,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "gave_up_or_loop",
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-04-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "4e9c275e232f10211ec88644d106a1f6a1b60e06",
        "engine_tool_parser": "qwen3_coder",
        "exact_model_id": "ibm-granite/granite-4.2-3b",
        "native_documented_call_format": "openai tool_calls via engine tool parser qwen3_coder",
        "plan_sha256": "ba105e1f98f38e5f7032392f3ec701cae4ab2ad9d3b976cf14d0b0a4425a81eb",
        "provider": "local vLLM served by @logic on bariton (#10256 standard set; #10380 hard set, identical argv)",
        "requested_name": "Granite 4.2 3B",
        "revision": "4e9c275e232f10211ec88644d106a1f6a1b60e06",
        "thinking_mode": "off"
      },
      "tiers": {
        "all": 3,
        "original": 2,
        "renamed": 2
      },
      "total_tokens": 2601502,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.3888888888888889,
          "passed": 28,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.4166666666666667,
          "passed": 30,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.3055555555555556,
          "passed": 11,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.4351851851851852,
          "passed": 47,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 0.75,
          "passed": 18,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 0.875,
          "passed": 21,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.7916666666666666,
          "passed": 19,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 0.875,
          "passed": 21,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "tiny",
        "class_label": "Open \u00b7 3.0B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/ibm-granite/granite-4.2-3b/blob/4e9c275e232f10211ec88644d106a1f6a1b60e06/README.md",
        "maker": "IBM",
        "params_active": 3.0,
        "params_total": 3.0,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "extra_call": 4,
        "no_call": 1,
        "wrong_args": 12
      },
      "gpu": {
        "card": "bariton GPU 2 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 724.3,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 32
      },
      "latency_seconds": {
        "mean": 61.37576334731919,
        "total": 8838.109922013964
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.8819444444444444,
        "passed": 127,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "no_call",
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "4e9c275e232f10211ec88644d106a1f6a1b60e06",
        "engine_tool_parser": "qwen3_coder",
        "exact_model_id": "ibm-granite/granite-4.2-3b",
        "native_documented_call_format": "openai tool_calls via engine tool parser qwen3_coder",
        "plan_sha256": "ba105e1f98f38e5f7032392f3ec701cae4ab2ad9d3b976cf14d0b0a4425a81eb",
        "provider": "local vLLM served by @logic on bariton (#10256 standard set; #10380 hard set, identical argv)",
        "requested_name": "Granite 4.2 3B",
        "revision": "4e9c275e232f10211ec88644d106a1f6a1b60e06",
        "thinking_mode": "on"
      },
      "tiers": {
        "all": 1,
        "original": 1,
        "renamed": 1
      },
      "total_tokens": 4304813,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.8472222222222222,
          "passed": 61,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.9166666666666666,
          "passed": 66,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.9722222222222222,
          "passed": 35,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.8518518518518519,
          "passed": 92,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 0.75,
          "passed": 18,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 0.7916666666666666,
          "passed": 19,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 0.20833333333333334,
          "passed": 5,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 0.3333333333333333,
          "passed": 8,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.08333333333333333,
          "passed": 2,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 0.375,
          "passed": 9,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "small",
        "class_label": "Open \u00b7 8.0B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/ibm-granite/granite-4.2-8b/blob/a7042144125ce5ae64f89610e8289647fc000b9a/README.md",
        "maker": "IBM",
        "params_active": 8.0,
        "params_total": 8.0,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "extra_call": 14,
        "wrong_args": 68,
        "wrong_tool": 1
      },
      "gpu": {
        "card": "bariton GPU 2 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 289.6,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 16
      },
      "latency_seconds": {
        "mean": 23.2329170641736,
        "total": 3345.540057240998
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.4236111111111111,
        "passed": 61,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_tool",
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "a7042144125ce5ae64f89610e8289647fc000b9a",
        "engine_tool_parser": "qwen3_coder",
        "exact_model_id": "ibm-granite/granite-4.2-8b",
        "native_documented_call_format": "openai tool_calls via engine tool parser qwen3_coder",
        "plan_sha256": "3beb9ba5bb3c3d19abb7e9fba70c7515c49c57204320d8c207582d13091296f3",
        "provider": "local vLLM served by @logic on bariton (#10281 standard set; #10458 hard set, identical argv)",
        "requested_name": "Granite 4.2 8B",
        "revision": "a7042144125ce5ae64f89610e8289647fc000b9a",
        "thinking_mode": "off"
      },
      "tiers": {
        "all": 3,
        "original": 3,
        "renamed": 3
      },
      "total_tokens": 2353976,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.4166666666666667,
          "passed": 30,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.4305555555555556,
          "passed": 31,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.4166666666666667,
          "passed": 15,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.42592592592592593,
          "passed": 46,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 0.9583333333333334,
          "passed": 23,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 0.7916666666666666,
          "passed": 19,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.7391304347826086,
          "passed": 17,
          "scored": 23,
          "unscored": 1
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "small",
        "class_label": "Open \u00b7 8.0B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/ibm-granite/granite-4.2-8b/blob/a7042144125ce5ae64f89610e8289647fc000b9a/README.md",
        "maker": "IBM",
        "params_active": 8.0,
        "params_total": 8.0,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "wrong_args": 12
      },
      "gpu": {
        "card": "bariton GPU 2 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 1214.5,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 16
      },
      "latency_seconds": {
        "mean": 94.75323884334736,
        "total": 13644.46639344202
      },
      "note": "Not tiered: one of its 144 task runs is unscored. On a long-output task the server refused the request because the prompt plus the fixed 32,768-token answer budget exceeded its 69,632-token context (HTTP 400). Serving and configuration errors are not counted as model failures; the score is over the scored runs.",
      "overall": {
        "attempted": 144,
        "pass_rate": 0.916083916083916,
        "passed": 131,
        "scored": 143,
        "unscored": 1
      },
      "public_tasks": [
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "unscored",
          "tag": null,
          "task_id": "hard-long-output-01"
        }
      ],
      "rankable": false,
      "setup": {
        "chat_template_revision": "a7042144125ce5ae64f89610e8289647fc000b9a",
        "engine_tool_parser": "qwen3_coder",
        "exact_model_id": "ibm-granite/granite-4.2-8b",
        "native_documented_call_format": "openai tool_calls via engine tool parser qwen3_coder",
        "plan_sha256": "3beb9ba5bb3c3d19abb7e9fba70c7515c49c57204320d8c207582d13091296f3",
        "provider": "local vLLM served by @logic on bariton (#10281 standard set; #10458 hard set, identical argv)",
        "requested_name": "Granite 4.2 8B",
        "revision": "a7042144125ce5ae64f89610e8289647fc000b9a",
        "thinking_mode": "on"
      },
      "total_tokens": null,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.9577464788732394,
          "passed": 68,
          "scored": 71,
          "unscored": 1
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.875,
          "passed": 63,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {
        "infrastructure": 1
      },
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.9166666666666666,
          "passed": 33,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.9158878504672897,
          "passed": 98,
          "scored": 107,
          "unscored": 1
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 0.75,
          "passed": 18,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 0.8333333333333334,
          "passed": 20,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.0,
          "passed": 0,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 0.7083333333333334,
          "passed": 17,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "small",
        "class_label": "Open \u00b7 12B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/google/gemma-4-12B-it/blob/43c9d014b231474fa7212dbb48a762566c2b255d/README.md",
        "maker": "Google",
        "params_active": 11.95,
        "params_total": 11.95,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "extra_call": 7,
        "wrong_args": 34
      },
      "gpu": {
        "card": "bariton GPU 2 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 82.8,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 16
      },
      "latency_seconds": {
        "mean": 8.251553268250039,
        "total": 1188.2236706280055
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.7152777777777778,
        "passed": 103,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "43c9d014b231474fa7212dbb48a762566c2b255d",
        "engine_tool_parser": "gemma4",
        "exact_model_id": "google/gemma-4-12B-it",
        "native_documented_call_format": "openai tool_calls via engine tool parser gemma4",
        "plan_sha256": "3e4f6db18dd0ef1a02cd476b5a31f013c4b3efc9598fd56fcfa5d4bb48fea5da",
        "provider": "local vLLM served by @logic on bariton (#10304 standard set; #10528 hard set, identical argv)",
        "requested_name": "Gemma 4 12B",
        "revision": "43c9d014b231474fa7212dbb48a762566c2b255d",
        "thinking_mode": "off"
      },
      "tiers": {
        "all": 2,
        "original": 2,
        "renamed": 2
      },
      "total_tokens": 1798075,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.7083333333333334,
          "passed": 51,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.7222222222222222,
          "passed": 52,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.6666666666666666,
          "passed": 24,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.7314814814814815,
          "passed": 79,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.7083333333333334,
          "passed": 17,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "small",
        "class_label": "Open \u00b7 12B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/google/gemma-4-12B-it/blob/43c9d014b231474fa7212dbb48a762566c2b255d/README.md",
        "maker": "Google",
        "params_active": 11.95,
        "params_total": 11.95,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "extra_call": 1,
        "wrong_args": 6
      },
      "gpu": {
        "card": "bariton GPU 2 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 229.1,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 16
      },
      "latency_seconds": {
        "mean": 23.041120779222084,
        "total": 3317.92139220798
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.9513888888888888,
        "passed": 137,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "43c9d014b231474fa7212dbb48a762566c2b255d",
        "engine_tool_parser": "gemma4",
        "exact_model_id": "google/gemma-4-12B-it",
        "native_documented_call_format": "openai tool_calls via engine tool parser gemma4",
        "plan_sha256": "3e4f6db18dd0ef1a02cd476b5a31f013c4b3efc9598fd56fcfa5d4bb48fea5da",
        "provider": "local vLLM served by @logic on bariton (#10304 standard set; #10528 hard set, identical argv)",
        "requested_name": "Gemma 4 12B",
        "revision": "43c9d014b231474fa7212dbb48a762566c2b255d",
        "thinking_mode": "on"
      },
      "tiers": {
        "all": 1,
        "original": 1,
        "renamed": 1
      },
      "total_tokens": 2017522,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.9444444444444444,
          "passed": 68,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.9583333333333334,
          "passed": 69,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.9166666666666666,
          "passed": 33,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.9629629629629629,
          "passed": 104,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 0.875,
          "passed": 21,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 0.9583333333333334,
          "passed": 23,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.6666666666666666,
          "passed": 16,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 0.9166666666666666,
          "passed": 22,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "medium",
        "class_label": "Open \u00b7 24B MoE \u00b7 2.5B active",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/LiquidAI/LFM2.5-24B-A2B/blob/1769516ac588c4c558089c94f27d96b0dc67b183/README.md",
        "maker": "Liquid AI",
        "params_active": 2.45,
        "params_total": 24.0,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "bad_format": 8,
        "extra_call": 1,
        "wrong_args": 5
      },
      "gpu": {
        "card": "bariton GPU 1 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 201.0,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 16
      },
      "latency_seconds": {
        "mean": 21.22846334667368,
        "total": 3056.8987219210103
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.9027777777777778,
        "passed": 130,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "1769516ac588c4c558089c94f27d96b0dc67b183",
        "engine_tool_parser": "lfm2",
        "exact_model_id": "LiquidAI/LFM2.5-24B-A2B",
        "native_documented_call_format": "openai tool_calls via engine tool parser lfm2",
        "plan_sha256": "0052721efa1e7dd2b96455bf856e0ae05e5609b9c6080f68c18d566f94dc720c",
        "provider": "local vLLM served by @logic on bariton (#10154 standard set; #10561 hard set on GPU 1, identical argv)",
        "requested_name": "LFM2.5-24B-A2B",
        "revision": "1769516ac588c4c558089c94f27d96b0dc67b183",
        "thinking_mode": "on"
      },
      "tiers": {
        "all": 1,
        "original": 1,
        "renamed": 1
      },
      "total_tokens": 2110323,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.9166666666666666,
          "passed": 66,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.8888888888888888,
          "passed": 64,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.9444444444444444,
          "passed": 34,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.8888888888888888,
          "passed": 96,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 0.75,
          "passed": 18,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 0.7083333333333334,
          "passed": 17,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 0.75,
          "passed": 18,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.25,
          "passed": 6,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "medium",
        "class_label": "Open \u00b7 27B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
        "maker": "Alibaba",
        "params_active": 27.0,
        "params_total": 27.0,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "bad_format": 1,
        "extra_call": 6,
        "wrong_args": 30
      },
      "gpu": {
        "card": "bariton GPU 1 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 241.8,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 16
      },
      "latency_seconds": {
        "mean": 25.823696603722382,
        "total": 3718.6123109360233
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.7430555555555556,
        "passed": 107,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
        "engine_tool_parser": "qwen3_coder",
        "exact_model_id": "Qwen/Qwen3.8-27B",
        "native_documented_call_format": "openai tool_calls via engine tool parser qwen3_coder",
        "plan_sha256": "a4ccebebea3a294a9f9f553ac9c86aedf045ce6105cbe3e3622b8646b0dc75b2",
        "provider": "local vLLM served by @logic on bariton (#10154 standard set; #10522 hard set on GPU 1, identical argv)",
        "requested_name": "Qwen3.8-27B",
        "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
        "thinking_mode": "off"
      },
      "tiers": {
        "all": 1,
        "original": 1,
        "renamed": 1
      },
      "total_tokens": 2144354,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.75,
          "passed": 54,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.7361111111111112,
          "passed": 53,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.8611111111111112,
          "passed": 31,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.7037037037037037,
          "passed": 76,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 0.9583333333333334,
          "passed": 23,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 0.9583333333333334,
          "passed": 23,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 0.875,
          "passed": 21,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 0.875,
          "passed": 21,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.625,
          "passed": 15,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "medium",
        "class_label": "Open \u00b7 27B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/Qwen/Qwen3.8-27B/blob/1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0/README.md",
        "maker": "Alibaba",
        "params_active": 27.0,
        "params_total": 27.0,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "bad_format": 9,
        "extra_call": 3,
        "wrong_args": 5
      },
      "gpu": {
        "card": "bariton GPU 1 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 515.6,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 16
      },
      "latency_seconds": {
        "mean": 53.048402196402805,
        "total": 7638.969916282003
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.8819444444444444,
        "passed": 127,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
        "engine_tool_parser": "qwen3_coder",
        "exact_model_id": "Qwen/Qwen3.8-27B",
        "native_documented_call_format": "openai tool_calls via engine tool parser qwen3_coder",
        "plan_sha256": "a4ccebebea3a294a9f9f553ac9c86aedf045ce6105cbe3e3622b8646b0dc75b2",
        "provider": "local vLLM served by @logic on bariton (#10154 standard set; #10522 hard set on GPU 1, identical argv)",
        "requested_name": "Qwen3.8-27B",
        "revision": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
        "thinking_mode": "on"
      },
      "tiers": {
        "all": 1,
        "original": 1,
        "renamed": 1
      },
      "total_tokens": 2204967,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.8888888888888888,
          "passed": 64,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.875,
          "passed": 63,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.8888888888888888,
          "passed": 32,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.8796296296296297,
          "passed": 95,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 0.75,
          "passed": 18,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 0.8333333333333334,
          "passed": 20,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 0.625,
          "passed": 15,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 0.75,
          "passed": 18,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.125,
          "passed": 3,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 0.7916666666666666,
          "passed": 19,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "medium",
        "class_label": "Open \u00b7 31B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/google/gemma-4-31B-it/blob/842da3794eaa0b77d5f08bae87a17459d91ff475/README.md",
        "maker": "Google",
        "params_active": 30.7,
        "params_total": 30.7,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "bad_format": 20,
        "extra_call": 6,
        "wrong_args": 25
      },
      "gpu": {
        "card": "bariton GPU 2 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 280.9,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 16
      },
      "latency_seconds": {
        "mean": 29.182258737840407,
        "total": 4202.245258249019
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.6458333333333334,
        "passed": 93,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "extra_call",
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "wrong_args",
          "task_id": "hard-long-output-01-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
        "engine_tool_parser": "gemma4",
        "exact_model_id": "google/gemma-4-31B-it",
        "native_documented_call_format": "openai tool_calls via engine tool parser gemma4",
        "plan_sha256": "f361ce9fe943d0ff88dc3f8af1d5483ecbdce3e21af90c00508b6d0dc6368236",
        "provider": "local vLLM served by @logic on bariton (#10154 GPU 0 for qualification; #10267 GPU 2 for the standard runs; #10577 GPU 2 for the hard set; argv/image/template identical)",
        "requested_name": "Gemma 4 31B",
        "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
        "thinking_mode": "off"
      },
      "tiers": {
        "all": 2,
        "original": 1,
        "renamed": 1
      },
      "total_tokens": 1565256,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.625,
          "passed": 45,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.6666666666666666,
          "passed": 48,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 0.8333333333333334,
          "passed": 30,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.5833333333333334,
          "passed": 63,
          "scored": 108,
          "unscored": 0
        }
      }
    },
    {
      "categories": {
        "branching": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "cascading-recovery": {
          "attempted": 24,
          "pass_rate": 0.9583333333333334,
          "passed": 23,
          "scored": 24,
          "unscored": 0
        },
        "computed-args": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "long-chain": {
          "attempted": 24,
          "pass_rate": 1.0,
          "passed": 24,
          "scored": 24,
          "unscored": 0
        },
        "long-output": {
          "attempted": 24,
          "pass_rate": 0.75,
          "passed": 18,
          "scored": 24,
          "unscored": 0
        },
        "multi-constraint": {
          "attempted": 24,
          "pass_rate": 0.75,
          "passed": 18,
          "scored": 24,
          "unscored": 0
        }
      },
      "classification": {
        "class_key": "medium",
        "class_label": "Open \u00b7 31B",
        "classification_basis": "total_parameters",
        "classification_source": "https://huggingface.co/google/gemma-4-31B-it/blob/842da3794eaa0b77d5f08bae87a17459d91ff475/README.md",
        "maker": "Google",
        "params_active": 30.7,
        "params_total": 30.7,
        "precision": "BF16",
        "quantization": null
      },
      "cost_usd": null,
      "failure_tags": {
        "bad_format": 13
      },
      "gpu": {
        "card": "bariton GPU 2 (RTX PRO 6000, 96 GB)",
        "gpu_count": 1,
        "gpu_seconds": 608.2,
        "in_flight": 16,
        "rate_usd_per_gpu_hour": 2.49,
        "server_max_num_seqs": 16
      },
      "latency_seconds": {
        "mean": 63.260641525527944,
        "total": 9109.532379676024
      },
      "overall": {
        "attempted": 144,
        "pass_rate": 0.9097222222222222,
        "passed": 131,
        "scored": 144,
        "unscored": 0
      },
      "public_tasks": [
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": false,
          "repetition": 0,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": true,
          "repetition": 0,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": false,
          "repetition": 1,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": true,
          "repetition": 1,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-multi-constraint-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-01-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-multi-constraint-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-chain-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-02-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-01-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-cascading-recovery-03-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-computed-args-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-branching-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-02-renamed"
        },
        {
          "passed": false,
          "repetition": 2,
          "status": "scored",
          "tag": "bad_format",
          "task_id": "hard-long-output-04-renamed"
        },
        {
          "passed": true,
          "repetition": 2,
          "status": "scored",
          "tag": null,
          "task_id": "hard-long-output-01-renamed"
        }
      ],
      "rankable": true,
      "setup": {
        "chat_template_revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
        "engine_tool_parser": "gemma4",
        "exact_model_id": "google/gemma-4-31B-it",
        "native_documented_call_format": "openai tool_calls via engine tool parser gemma4",
        "plan_sha256": "f361ce9fe943d0ff88dc3f8af1d5483ecbdce3e21af90c00508b6d0dc6368236",
        "provider": "local vLLM served by @logic on bariton (#10154 GPU 0 for qualification; #10267 GPU 2 for the standard runs; #10577 GPU 2 for the hard set; argv/image/template identical)",
        "requested_name": "Gemma 4 31B",
        "revision": "842da3794eaa0b77d5f08bae87a17459d91ff475",
        "thinking_mode": "on"
      },
      "tiers": {
        "all": 1,
        "original": 1,
        "renamed": 1
      },
      "total_tokens": 1830591,
      "tracks": {
        "original": {
          "attempted": 72,
          "pass_rate": 0.9027777777777778,
          "passed": 65,
          "scored": 72,
          "unscored": 0
        },
        "renamed": {
          "attempted": 72,
          "pass_rate": 0.9166666666666666,
          "passed": 66,
          "scored": 72,
          "unscored": 0
        }
      },
      "unscored_reasons": {},
      "visibility": {
        "private": {
          "attempted": 36,
          "pass_rate": 1.0,
          "passed": 36,
          "scored": 36,
          "unscored": 0
        },
        "public": {
          "attempted": 108,
          "pass_rate": 0.8796296296296297,
          "passed": 95,
          "scored": 108,
          "unscored": 0
        }
      }
    }
  ]
}
