{
  "codes": {
    "F": "failed (scored)",
    "P": "passed",
    "U": "unscored (serving or configuration error, not a model failure)"
  },
  "generated_at": "2026-11-16T18:29:23+01:00",
  "order": "position i of every outcomes string is task_ids[i] of its track; public tasks only (private task outcomes are never published)",
  "tracks": {
    "original": {
      "runs": [
        {
          "outcomes": "FFFFPFFPPPPPFFPFPF",
          "repetition": 0,
          "report_id": "f6a742942de067fc3de4686e688c564e80be956f0bbeee5b08c5f96d12e160af",
          "seed": 0,
          "setup": "LFM3-1.7B",
          "thinking": "off"
        },
        {
          "outcomes": "FFFFPFFPPPPPPFPFFF",
          "repetition": 1,
          "report_id": "f6a742942de067fc3de4686e688c564e80be956f0bbeee5b08c5f96d12e160af",
          "seed": 1,
          "setup": "LFM3-1.7B",
          "thinking": "off"
        },
        {
          "outcomes": "FFFFFFFPPPPPFFFFFF",
          "repetition": 2,
          "report_id": "f6a742942de067fc3de4686e688c564e80be956f0bbeee5b08c5f96d12e160af",
          "seed": 2,
          "setup": "LFM3-1.7B",
          "thinking": "off"
        },
        {
          "outcomes": "PFFPPPFFPPPPFFPFPF",
          "repetition": 0,
          "report_id": "49054f114d4aa8ab00b0c970a466e93e09bdb8a8a2b13e094cb3f89fa4ccb6f5",
          "seed": 0,
          "setup": "LFM3-1.7B",
          "thinking": "on"
        },
        {
          "outcomes": "PFFPPPFFPPPPPFPFPF",
          "repetition": 1,
          "report_id": "49054f114d4aa8ab00b0c970a466e93e09bdb8a8a2b13e094cb3f89fa4ccb6f5",
          "seed": 1,
          "setup": "LFM3-1.7B",
          "thinking": "on"
        },
        {
          "outcomes": "PPFPPPFFPPPPFFPFPF",
          "repetition": 2,
          "report_id": "49054f114d4aa8ab00b0c970a466e93e09bdb8a8a2b13e094cb3f89fa4ccb6f5",
          "seed": 2,
          "setup": "LFM3-1.7B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPFFFPPPPPFPFPF",
          "repetition": 0,
          "report_id": "6438dc0d15728a182dc3216c2f94299e8263235b2e728a125ff9f8cadbdc76ed",
          "seed": 0,
          "setup": "Gemma 4 E2B",
          "thinking": "on"
        },
        {
          "outcomes": "PPFFPPFFPPPPPPPFPF",
          "repetition": 1,
          "report_id": "6438dc0d15728a182dc3216c2f94299e8263235b2e728a125ff9f8cadbdc76ed",
          "seed": 1,
          "setup": "Gemma 4 E2B",
          "thinking": "on"
        },
        {
          "outcomes": "PFFPPPFFPPPPPFPFPF",
          "repetition": 2,
          "report_id": "6438dc0d15728a182dc3216c2f94299e8263235b2e728a125ff9f8cadbdc76ed",
          "seed": 2,
          "setup": "Gemma 4 E2B",
          "thinking": "on"
        },
        {
          "outcomes": "FFFFPFFFPPPPPPPFFF",
          "repetition": 0,
          "report_id": "53112cc9e51a3c84619a497052868f2be47fd3bfcd44b4eb17af43f0f08311a9",
          "seed": 0,
          "setup": "MiniCPM5-2B",
          "thinking": "off"
        },
        {
          "outcomes": "PFFFPPFFFPFPFPFFPF",
          "repetition": 1,
          "report_id": "53112cc9e51a3c84619a497052868f2be47fd3bfcd44b4eb17af43f0f08311a9",
          "seed": 1,
          "setup": "MiniCPM5-2B",
          "thinking": "off"
        },
        {
          "outcomes": "PFFFPFFFPPPPFPPFFF",
          "repetition": 2,
          "report_id": "53112cc9e51a3c84619a497052868f2be47fd3bfcd44b4eb17af43f0f08311a9",
          "seed": 2,
          "setup": "MiniCPM5-2B",
          "thinking": "off"
        },
        {
          "outcomes": "PFPPPPPPPPPPPPPPPF",
          "repetition": 0,
          "report_id": "27a84c9ac606f29120025ba5146dab10d8f79ef462e0349180345e8f555e1039",
          "seed": 0,
          "setup": "MiniCPM5-2B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPFPPPPPPPPPPF",
          "repetition": 1,
          "report_id": "27a84c9ac606f29120025ba5146dab10d8f79ef462e0349180345e8f555e1039",
          "seed": 1,
          "setup": "MiniCPM5-2B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPFPPFPPPPPPPPPPF",
          "repetition": 2,
          "report_id": "27a84c9ac606f29120025ba5146dab10d8f79ef462e0349180345e8f555e1039",
          "seed": 2,
          "setup": "MiniCPM5-2B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPFFFPPPFPPFPPFFF",
          "repetition": 0,
          "report_id": "14a1670c43f9b70edb6d260fd3652c7187b98c97fd041848fce7f700d36730ff",
          "seed": 0,
          "setup": "Qwen3.5-2B",
          "thinking": "off"
        },
        {
          "outcomes": "PFFFPPPPPPFPPPPPFF",
          "repetition": 1,
          "report_id": "14a1670c43f9b70edb6d260fd3652c7187b98c97fd041848fce7f700d36730ff",
          "seed": 1,
          "setup": "Qwen3.5-2B",
          "thinking": "off"
        },
        {
          "outcomes": "PFPFFFFFPPPPPPFPFF",
          "repetition": 2,
          "report_id": "14a1670c43f9b70edb6d260fd3652c7187b98c97fd041848fce7f700d36730ff",
          "seed": 2,
          "setup": "Qwen3.5-2B",
          "thinking": "off"
        },
        {
          "outcomes": "PFPFFPPPPPPPPPPFFF",
          "repetition": 0,
          "report_id": "21f3ef13e659ef514e71016f70155fa31e593844002b71343fc4d06cde56b3fe",
          "seed": 0,
          "setup": "Qwen3.5-2B",
          "thinking": "on"
        },
        {
          "outcomes": "PFPFPFFPPPPPPPPFPF",
          "repetition": 1,
          "report_id": "21f3ef13e659ef514e71016f70155fa31e593844002b71343fc4d06cde56b3fe",
          "seed": 1,
          "setup": "Qwen3.5-2B",
          "thinking": "on"
        },
        {
          "outcomes": "PFPFPPFPPPPPPFPFPF",
          "repetition": 2,
          "report_id": "21f3ef13e659ef514e71016f70155fa31e593844002b71343fc4d06cde56b3fe",
          "seed": 2,
          "setup": "Qwen3.5-2B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPFPPPPPPPPPPPFPP",
          "repetition": 0,
          "report_id": "5a64e9a304ce5c22dfd4735c60096c3fe6de65503565fd9c137b08a2bde90f1e",
          "seed": 0,
          "setup": "Qwen3.5 4B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPPPPPPPPPPPPPFPP",
          "repetition": 1,
          "report_id": "5a64e9a304ce5c22dfd4735c60096c3fe6de65503565fd9c137b08a2bde90f1e",
          "seed": 1,
          "setup": "Qwen3.5 4B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPPPPPPPPPPPPPFPP",
          "repetition": 2,
          "report_id": "5a64e9a304ce5c22dfd4735c60096c3fe6de65503565fd9c137b08a2bde90f1e",
          "seed": 2,
          "setup": "Qwen3.5 4B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPPPPPPPPPPPPPFFF",
          "repetition": 0,
          "report_id": "10f52280f44dd977a0733509d0804f1816b0e0644c25d892d70f1f66e9802318",
          "seed": 0,
          "setup": "Qwen3.5 4B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPFPPPPPPPPFFPPFF",
          "repetition": 1,
          "report_id": "10f52280f44dd977a0733509d0804f1816b0e0644c25d892d70f1f66e9802318",
          "seed": 1,
          "setup": "Qwen3.5 4B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPFPPPPPPFPPFPFPF",
          "repetition": 2,
          "report_id": "10f52280f44dd977a0733509d0804f1816b0e0644c25d892d70f1f66e9802318",
          "seed": 2,
          "setup": "Qwen3.5 4B",
          "thinking": "on"
        },
        {
          "outcomes": "FFFFFFFFPPPPPPPFFF",
          "repetition": 0,
          "report_id": "6e10091742dfb5f0288682ffb0410d23bff860b2d03558c320630b957452baf6",
          "seed": 0,
          "setup": "Raon-1 nano",
          "thinking": "off"
        },
        {
          "outcomes": "FFFFFFFFFFPFPPPFFF",
          "repetition": 1,
          "report_id": "6e10091742dfb5f0288682ffb0410d23bff860b2d03558c320630b957452baf6",
          "seed": 1,
          "setup": "Raon-1 nano",
          "thinking": "off"
        },
        {
          "outcomes": "FFFFFFFFPPPFPFPFFF",
          "repetition": 2,
          "report_id": "6e10091742dfb5f0288682ffb0410d23bff860b2d03558c320630b957452baf6",
          "seed": 2,
          "setup": "Raon-1 nano",
          "thinking": "off"
        },
        {
          "outcomes": "FPPPPFFPPPPPPFPFFF",
          "repetition": 0,
          "report_id": "d2f720cf1b957dd753d1e58d2e828032b15b14a9ab92d897881a20200f6195e1",
          "seed": 0,
          "setup": "Raon-1 nano",
          "thinking": "on"
        },
        {
          "outcomes": "PPPFPPFPPPPPPPPFPF",
          "repetition": 1,
          "report_id": "d2f720cf1b957dd753d1e58d2e828032b15b14a9ab92d897881a20200f6195e1",
          "seed": 1,
          "setup": "Raon-1 nano",
          "thinking": "on"
        },
        {
          "outcomes": "FPPPPFFPPPPPPPFFFF",
          "repetition": 2,
          "report_id": "d2f720cf1b957dd753d1e58d2e828032b15b14a9ab92d897881a20200f6195e1",
          "seed": 2,
          "setup": "Raon-1 nano",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPFPPPPPPPPFPF",
          "repetition": 0,
          "report_id": "24f1c17bca4f3b38537e72c6986fabd7b26ff4299c7e541839e3202593272272",
          "seed": 0,
          "setup": "Gemma 4 E2B (tool-result cue template)",
          "thinking": "off"
        },
        {
          "outcomes": "PFPPPFFPPPPPPPPFPF",
          "repetition": 1,
          "report_id": "24f1c17bca4f3b38537e72c6986fabd7b26ff4299c7e541839e3202593272272",
          "seed": 1,
          "setup": "Gemma 4 E2B (tool-result cue template)",
          "thinking": "off"
        },
        {
          "outcomes": "PPPPPPFPPPPPPPPFPF",
          "repetition": 2,
          "report_id": "24f1c17bca4f3b38537e72c6986fabd7b26ff4299c7e541839e3202593272272",
          "seed": 2,
          "setup": "Gemma 4 E2B (tool-result cue template)",
          "thinking": "off"
        },
        {
          "outcomes": "FFFFFFFPFPPPPFFFPF",
          "repetition": 0,
          "report_id": "96415472354b896e265793e2365fc5b2bebc5594968e98f2c325bb3c39f43339",
          "seed": 0,
          "setup": "Granite 4.2 3B",
          "thinking": "off"
        },
        {
          "outcomes": "PFFFPFFPPFPPPPFFPF",
          "repetition": 1,
          "report_id": "96415472354b896e265793e2365fc5b2bebc5594968e98f2c325bb3c39f43339",
          "seed": 1,
          "setup": "Granite 4.2 3B",
          "thinking": "off"
        },
        {
          "outcomes": "FFFFPFFPPPPPPPFFFF",
          "repetition": 2,
          "report_id": "96415472354b896e265793e2365fc5b2bebc5594968e98f2c325bb3c39f43339",
          "seed": 2,
          "setup": "Granite 4.2 3B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPFPPPPFPPPPFPPPF",
          "repetition": 0,
          "report_id": "d22fa30c9004d04be7e72181e46b3f7fdd3ec9120580253f3f1e57f22d72c827",
          "seed": 0,
          "setup": "Granite 4.2 3B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPPPPPPFFFPPP",
          "repetition": 1,
          "report_id": "d22fa30c9004d04be7e72181e46b3f7fdd3ec9120580253f3f1e57f22d72c827",
          "seed": 1,
          "setup": "Granite 4.2 3B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPFPPPPPPPPPFPPPF",
          "repetition": 2,
          "report_id": "d22fa30c9004d04be7e72181e46b3f7fdd3ec9120580253f3f1e57f22d72c827",
          "seed": 2,
          "setup": "Granite 4.2 3B",
          "thinking": "on"
        },
        {
          "outcomes": "FPFFFPFPPFPPPPPFFF",
          "repetition": 0,
          "report_id": "79f5324a24db4cc6683f5865e00eb7d8331fe137d891fbdf13dfa98cb58be2d3",
          "seed": 0,
          "setup": "Granite 4.2 8B",
          "thinking": "off"
        },
        {
          "outcomes": "FFFPFFFPPFPPPPFFFF",
          "repetition": 1,
          "report_id": "79f5324a24db4cc6683f5865e00eb7d8331fe137d891fbdf13dfa98cb58be2d3",
          "seed": 1,
          "setup": "Granite 4.2 8B",
          "thinking": "off"
        },
        {
          "outcomes": "FFFFPFFPPFPPPPFFFF",
          "repetition": 2,
          "report_id": "79f5324a24db4cc6683f5865e00eb7d8331fe137d891fbdf13dfa98cb58be2d3",
          "seed": 2,
          "setup": "Granite 4.2 8B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPPPPFPPPPPPPPFPP",
          "repetition": 0,
          "report_id": "511dae62ca5252589cc84079f21e12e5a4e9da5859b8010ed9dba2db800e6cc2",
          "seed": 0,
          "setup": "Granite 4.2 8B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPPPPPPPPPPPP",
          "repetition": 1,
          "report_id": "511dae62ca5252589cc84079f21e12e5a4e9da5859b8010ed9dba2db800e6cc2",
          "seed": 1,
          "setup": "Granite 4.2 8B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPPPPPPPPPUPP",
          "repetition": 2,
          "report_id": "511dae62ca5252589cc84079f21e12e5a4e9da5859b8010ed9dba2db800e6cc2",
          "seed": 2,
          "setup": "Granite 4.2 8B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPFPPPPFPPPPPFFF",
          "repetition": 0,
          "report_id": "87114d424d910acc4d84092ad931c0475cb0d5065dd1225e549e670a2d5b032c",
          "seed": 0,
          "setup": "Gemma 4 12B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPPFPPPPFPPPPPFFF",
          "repetition": 1,
          "report_id": "87114d424d910acc4d84092ad931c0475cb0d5065dd1225e549e670a2d5b032c",
          "seed": 1,
          "setup": "Gemma 4 12B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPPFPPPPFPPPPPFFF",
          "repetition": 2,
          "report_id": "87114d424d910acc4d84092ad931c0475cb0d5065dd1225e549e670a2d5b032c",
          "seed": 2,
          "setup": "Gemma 4 12B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPPPPPPPPPPPPPFPP",
          "repetition": 0,
          "report_id": "e03a41c8c729e138a1fb4130837439a8db729ee21829c5ba0d89b80306493c30",
          "seed": 0,
          "setup": "Gemma 4 12B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPPPPPPPPPPPP",
          "repetition": 1,
          "report_id": "e03a41c8c729e138a1fb4130837439a8db729ee21829c5ba0d89b80306493c30",
          "seed": 1,
          "setup": "Gemma 4 12B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPPPPPPPPPFPF",
          "repetition": 2,
          "report_id": "e03a41c8c729e138a1fb4130837439a8db729ee21829c5ba0d89b80306493c30",
          "seed": 2,
          "setup": "Gemma 4 12B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPPPPPPPPPPPF",
          "repetition": 0,
          "report_id": "9918f896f7234fe98b21795fb4937fb29b7dfe49b233ddc1bdda4c5881fb2c8e",
          "seed": 0,
          "setup": "LFM2.5-24B-A2B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPPPPPPPFPPPF",
          "repetition": 1,
          "report_id": "9918f896f7234fe98b21795fb4937fb29b7dfe49b233ddc1bdda4c5881fb2c8e",
          "seed": 1,
          "setup": "LFM2.5-24B-A2B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPPPPPPPFPPPF",
          "repetition": 2,
          "report_id": "9918f896f7234fe98b21795fb4937fb29b7dfe49b233ddc1bdda4c5881fb2c8e",
          "seed": 2,
          "setup": "LFM2.5-24B-A2B",
          "thinking": "on"
        },
        {
          "outcomes": "PPFPFPPPPFPPPPPFFP",
          "repetition": 0,
          "report_id": "33b2fc529fb2de88831d43e81b9afd563c90d7d8dbded7d334a2ac02081daff4",
          "seed": 0,
          "setup": "Qwen3.8-27B",
          "thinking": "off"
        },
        {
          "outcomes": "PPFPFPPPPFPPPPPFFP",
          "repetition": 1,
          "report_id": "33b2fc529fb2de88831d43e81b9afd563c90d7d8dbded7d334a2ac02081daff4",
          "seed": 1,
          "setup": "Qwen3.8-27B",
          "thinking": "off"
        },
        {
          "outcomes": "PPFPPFPPPFPPPPPFFP",
          "repetition": 2,
          "report_id": "33b2fc529fb2de88831d43e81b9afd563c90d7d8dbded7d334a2ac02081daff4",
          "seed": 2,
          "setup": "Qwen3.8-27B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPPPPFPPPPPPFPPPF",
          "repetition": 0,
          "report_id": "604dc3902cff19025cbfe96022094ed0376406a03797afe689cb8227ffe6786c",
          "seed": 0,
          "setup": "Qwen3.8-27B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPPPPPPPPPFPF",
          "repetition": 1,
          "report_id": "604dc3902cff19025cbfe96022094ed0376406a03797afe689cb8227ffe6786c",
          "seed": 1,
          "setup": "Qwen3.8-27B",
          "thinking": "on"
        },
        {
          "outcomes": "PPFPPPPPPPPPPPPPPF",
          "repetition": 2,
          "report_id": "604dc3902cff19025cbfe96022094ed0376406a03797afe689cb8227ffe6786c",
          "seed": 2,
          "setup": "Qwen3.8-27B",
          "thinking": "on"
        },
        {
          "outcomes": "PPFPFPPFPFPPPFPFFF",
          "repetition": 0,
          "report_id": "f4b2e4a175cbb720ef525cacf7c1ce04ae09083a794457d2af61fbcf1f78e024",
          "seed": 0,
          "setup": "Gemma 4 31B",
          "thinking": "off"
        },
        {
          "outcomes": "PPFPFPPFPFPPPFPFFF",
          "repetition": 1,
          "report_id": "f4b2e4a175cbb720ef525cacf7c1ce04ae09083a794457d2af61fbcf1f78e024",
          "seed": 1,
          "setup": "Gemma 4 31B",
          "thinking": "off"
        },
        {
          "outcomes": "PPFPFPPFPFPPPFPFFF",
          "repetition": 2,
          "report_id": "f4b2e4a175cbb720ef525cacf7c1ce04ae09083a794457d2af61fbcf1f78e024",
          "seed": 2,
          "setup": "Gemma 4 31B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPPPPPFPPPPPPPPPF",
          "repetition": 0,
          "report_id": "a92b0fc63f5e4d1d3c6ad852aca000c3a793ff08d84ee076b0ab1367a5930506",
          "seed": 0,
          "setup": "Gemma 4 31B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPFPPPPPFPPPF",
          "repetition": 1,
          "report_id": "a92b0fc63f5e4d1d3c6ad852aca000c3a793ff08d84ee076b0ab1367a5930506",
          "seed": 1,
          "setup": "Gemma 4 31B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPFPPPPPPPPPF",
          "repetition": 2,
          "report_id": "a92b0fc63f5e4d1d3c6ad852aca000c3a793ff08d84ee076b0ab1367a5930506",
          "seed": 2,
          "setup": "Gemma 4 31B",
          "thinking": "on"
        }
      ],
      "task_ids": [
        "hard-long-chain-01",
        "hard-long-chain-02",
        "hard-long-chain-03",
        "hard-computed-args-01",
        "hard-computed-args-02",
        "hard-computed-args-03",
        "hard-computed-args-04",
        "hard-multi-constraint-03",
        "hard-multi-constraint-04",
        "hard-branching-01",
        "hard-branching-03",
        "hard-branching-04",
        "hard-cascading-recovery-01",
        "hard-cascading-recovery-02",
        "hard-cascading-recovery-03",
        "hard-long-output-01",
        "hard-long-output-02",
        "hard-long-output-04"
      ],
      "tasks": 18
    },
    "renamed": {
      "runs": [
        {
          "outcomes": "FFFFFFFFPPPPFFFFFF",
          "repetition": 0,
          "report_id": "f6a742942de067fc3de4686e688c564e80be956f0bbeee5b08c5f96d12e160af",
          "seed": 0,
          "setup": "LFM3-1.7B",
          "thinking": "off"
        },
        {
          "outcomes": "FFFFFFFFPPPPFFFFFF",
          "repetition": 1,
          "report_id": "f6a742942de067fc3de4686e688c564e80be956f0bbeee5b08c5f96d12e160af",
          "seed": 1,
          "setup": "LFM3-1.7B",
          "thinking": "off"
        },
        {
          "outcomes": "FFFFFFFFPPPPFFFFPF",
          "repetition": 2,
          "report_id": "f6a742942de067fc3de4686e688c564e80be956f0bbeee5b08c5f96d12e160af",
          "seed": 2,
          "setup": "LFM3-1.7B",
          "thinking": "off"
        },
        {
          "outcomes": "FPFPPPFPPPPPFFPFPF",
          "repetition": 0,
          "report_id": "49054f114d4aa8ab00b0c970a466e93e09bdb8a8a2b13e094cb3f89fa4ccb6f5",
          "seed": 0,
          "setup": "LFM3-1.7B",
          "thinking": "on"
        },
        {
          "outcomes": "FFFPPPFFPPPPPFPFFF",
          "repetition": 1,
          "report_id": "49054f114d4aa8ab00b0c970a466e93e09bdb8a8a2b13e094cb3f89fa4ccb6f5",
          "seed": 1,
          "setup": "LFM3-1.7B",
          "thinking": "on"
        },
        {
          "outcomes": "PPFPPPFFPPPPFFPPPF",
          "repetition": 2,
          "report_id": "49054f114d4aa8ab00b0c970a466e93e09bdb8a8a2b13e094cb3f89fa4ccb6f5",
          "seed": 2,
          "setup": "LFM3-1.7B",
          "thinking": "on"
        },
        {
          "outcomes": "PFFPPFFPPPPPPFPFPF",
          "repetition": 0,
          "report_id": "6438dc0d15728a182dc3216c2f94299e8263235b2e728a125ff9f8cadbdc76ed",
          "seed": 0,
          "setup": "Gemma 4 E2B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPFPPPPPPPPFPF",
          "repetition": 1,
          "report_id": "6438dc0d15728a182dc3216c2f94299e8263235b2e728a125ff9f8cadbdc76ed",
          "seed": 1,
          "setup": "Gemma 4 E2B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPFFPPPPPPPFPF",
          "repetition": 2,
          "report_id": "6438dc0d15728a182dc3216c2f94299e8263235b2e728a125ff9f8cadbdc76ed",
          "seed": 2,
          "setup": "Gemma 4 E2B",
          "thinking": "on"
        },
        {
          "outcomes": "FFFFFFFFFPPPFPFFFF",
          "repetition": 0,
          "report_id": "53112cc9e51a3c84619a497052868f2be47fd3bfcd44b4eb17af43f0f08311a9",
          "seed": 0,
          "setup": "MiniCPM5-2B",
          "thinking": "off"
        },
        {
          "outcomes": "FFFFFFFFFFPPPPFFFF",
          "repetition": 1,
          "report_id": "53112cc9e51a3c84619a497052868f2be47fd3bfcd44b4eb17af43f0f08311a9",
          "seed": 1,
          "setup": "MiniCPM5-2B",
          "thinking": "off"
        },
        {
          "outcomes": "FFFFFFFFFPPPPPFFFF",
          "repetition": 2,
          "report_id": "53112cc9e51a3c84619a497052868f2be47fd3bfcd44b4eb17af43f0f08311a9",
          "seed": 2,
          "setup": "MiniCPM5-2B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPFPPFPPPPPFPPPPF",
          "repetition": 0,
          "report_id": "27a84c9ac606f29120025ba5146dab10d8f79ef462e0349180345e8f555e1039",
          "seed": 0,
          "setup": "MiniCPM5-2B",
          "thinking": "on"
        },
        {
          "outcomes": "PPFFPFPPPPPPPPPPPF",
          "repetition": 1,
          "report_id": "27a84c9ac606f29120025ba5146dab10d8f79ef462e0349180345e8f555e1039",
          "seed": 1,
          "setup": "MiniCPM5-2B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPPPPPPPPPFPF",
          "repetition": 2,
          "report_id": "27a84c9ac606f29120025ba5146dab10d8f79ef462e0349180345e8f555e1039",
          "seed": 2,
          "setup": "MiniCPM5-2B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPFPFFPFFPPPPFFFF",
          "repetition": 0,
          "report_id": "14a1670c43f9b70edb6d260fd3652c7187b98c97fd041848fce7f700d36730ff",
          "seed": 0,
          "setup": "Qwen3.5-2B",
          "thinking": "off"
        },
        {
          "outcomes": "PFFFPPFPPPPPPPPFFF",
          "repetition": 1,
          "report_id": "14a1670c43f9b70edb6d260fd3652c7187b98c97fd041848fce7f700d36730ff",
          "seed": 1,
          "setup": "Qwen3.5-2B",
          "thinking": "off"
        },
        {
          "outcomes": "PFPFFFFPFPPFPPPFFF",
          "repetition": 2,
          "report_id": "14a1670c43f9b70edb6d260fd3652c7187b98c97fd041848fce7f700d36730ff",
          "seed": 2,
          "setup": "Qwen3.5-2B",
          "thinking": "off"
        },
        {
          "outcomes": "FPPFPPFPFPPPPPPFFF",
          "repetition": 0,
          "report_id": "21f3ef13e659ef514e71016f70155fa31e593844002b71343fc4d06cde56b3fe",
          "seed": 0,
          "setup": "Qwen3.5-2B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPFPPFPPPPPPPPFFF",
          "repetition": 1,
          "report_id": "21f3ef13e659ef514e71016f70155fa31e593844002b71343fc4d06cde56b3fe",
          "seed": 1,
          "setup": "Qwen3.5-2B",
          "thinking": "on"
        },
        {
          "outcomes": "PFPFPPFPPPPPFPFPPF",
          "repetition": 2,
          "report_id": "21f3ef13e659ef514e71016f70155fa31e593844002b71343fc4d06cde56b3fe",
          "seed": 2,
          "setup": "Qwen3.5-2B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPFPPPPPPPPPPPFPP",
          "repetition": 0,
          "report_id": "5a64e9a304ce5c22dfd4735c60096c3fe6de65503565fd9c137b08a2bde90f1e",
          "seed": 0,
          "setup": "Qwen3.5 4B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPFPPPPPPPPPFPFPF",
          "repetition": 1,
          "report_id": "5a64e9a304ce5c22dfd4735c60096c3fe6de65503565fd9c137b08a2bde90f1e",
          "seed": 1,
          "setup": "Qwen3.5 4B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPPPPPPPPPPPFPFPF",
          "repetition": 2,
          "report_id": "5a64e9a304ce5c22dfd4735c60096c3fe6de65503565fd9c137b08a2bde90f1e",
          "seed": 2,
          "setup": "Qwen3.5 4B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPPPPPPPPPPPFPFPF",
          "repetition": 0,
          "report_id": "10f52280f44dd977a0733509d0804f1816b0e0644c25d892d70f1f66e9802318",
          "seed": 0,
          "setup": "Qwen3.5 4B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPPPPPPPPPPPF",
          "repetition": 1,
          "report_id": "10f52280f44dd977a0733509d0804f1816b0e0644c25d892d70f1f66e9802318",
          "seed": 1,
          "setup": "Qwen3.5 4B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPFPPPPPPPFPF",
          "repetition": 2,
          "report_id": "10f52280f44dd977a0733509d0804f1816b0e0644c25d892d70f1f66e9802318",
          "seed": 2,
          "setup": "Qwen3.5 4B",
          "thinking": "on"
        },
        {
          "outcomes": "FFFFFFFFFFPFPFPFFF",
          "repetition": 0,
          "report_id": "6e10091742dfb5f0288682ffb0410d23bff860b2d03558c320630b957452baf6",
          "seed": 0,
          "setup": "Raon-1 nano",
          "thinking": "off"
        },
        {
          "outcomes": "FFFFPFFFFPPFPPPFFF",
          "repetition": 1,
          "report_id": "6e10091742dfb5f0288682ffb0410d23bff860b2d03558c320630b957452baf6",
          "seed": 1,
          "setup": "Raon-1 nano",
          "thinking": "off"
        },
        {
          "outcomes": "FFFFFFFFFFPFPPPFFF",
          "repetition": 2,
          "report_id": "6e10091742dfb5f0288682ffb0410d23bff860b2d03558c320630b957452baf6",
          "seed": 2,
          "setup": "Raon-1 nano",
          "thinking": "off"
        },
        {
          "outcomes": "FFPFPFFPPPPPPPPFPF",
          "repetition": 0,
          "report_id": "d2f720cf1b957dd753d1e58d2e828032b15b14a9ab92d897881a20200f6195e1",
          "seed": 0,
          "setup": "Raon-1 nano",
          "thinking": "on"
        },
        {
          "outcomes": "FPPPPFFPPPPPPPFFPF",
          "repetition": 1,
          "report_id": "d2f720cf1b957dd753d1e58d2e828032b15b14a9ab92d897881a20200f6195e1",
          "seed": 1,
          "setup": "Raon-1 nano",
          "thinking": "on"
        },
        {
          "outcomes": "PPPFPPFPPPPPPPPFPF",
          "repetition": 2,
          "report_id": "d2f720cf1b957dd753d1e58d2e828032b15b14a9ab92d897881a20200f6195e1",
          "seed": 2,
          "setup": "Raon-1 nano",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPFPPPPPPFPFPF",
          "repetition": 0,
          "report_id": "24f1c17bca4f3b38537e72c6986fabd7b26ff4299c7e541839e3202593272272",
          "seed": 0,
          "setup": "Gemma 4 E2B (tool-result cue template)",
          "thinking": "off"
        },
        {
          "outcomes": "PPFPPPFPPPPPPPPFPF",
          "repetition": 1,
          "report_id": "24f1c17bca4f3b38537e72c6986fabd7b26ff4299c7e541839e3202593272272",
          "seed": 1,
          "setup": "Gemma 4 E2B (tool-result cue template)",
          "thinking": "off"
        },
        {
          "outcomes": "PPPPPPFPPPPPPPPFFF",
          "repetition": 2,
          "report_id": "24f1c17bca4f3b38537e72c6986fabd7b26ff4299c7e541839e3202593272272",
          "seed": 2,
          "setup": "Gemma 4 E2B (tool-result cue template)",
          "thinking": "off"
        },
        {
          "outcomes": "FFPFPFFPPPPFPPFFPF",
          "repetition": 0,
          "report_id": "96415472354b896e265793e2365fc5b2bebc5594968e98f2c325bb3c39f43339",
          "seed": 0,
          "setup": "Granite 4.2 3B",
          "thinking": "off"
        },
        {
          "outcomes": "FPFFPFFPPPPFFPFFFF",
          "repetition": 1,
          "report_id": "96415472354b896e265793e2365fc5b2bebc5594968e98f2c325bb3c39f43339",
          "seed": 1,
          "setup": "Granite 4.2 3B",
          "thinking": "off"
        },
        {
          "outcomes": "FPPFPFFPFPPPFPFFFF",
          "repetition": 2,
          "report_id": "96415472354b896e265793e2365fc5b2bebc5594968e98f2c325bb3c39f43339",
          "seed": 2,
          "setup": "Granite 4.2 3B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPPPPPPPPPPPFPPPP",
          "repetition": 0,
          "report_id": "d22fa30c9004d04be7e72181e46b3f7fdd3ec9120580253f3f1e57f22d72c827",
          "seed": 0,
          "setup": "Granite 4.2 3B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPFPPPPPPPPPPPFPF",
          "repetition": 1,
          "report_id": "d22fa30c9004d04be7e72181e46b3f7fdd3ec9120580253f3f1e57f22d72c827",
          "seed": 1,
          "setup": "Granite 4.2 3B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPPFPPPPPPFPP",
          "repetition": 2,
          "report_id": "d22fa30c9004d04be7e72181e46b3f7fdd3ec9120580253f3f1e57f22d72c827",
          "seed": 2,
          "setup": "Granite 4.2 3B",
          "thinking": "on"
        },
        {
          "outcomes": "FPFPFFFFPFPPPPFFPF",
          "repetition": 0,
          "report_id": "79f5324a24db4cc6683f5865e00eb7d8331fe137d891fbdf13dfa98cb58be2d3",
          "seed": 0,
          "setup": "Granite 4.2 8B",
          "thinking": "off"
        },
        {
          "outcomes": "PFFFFPFFPFPPPPFPFF",
          "repetition": 1,
          "report_id": "79f5324a24db4cc6683f5865e00eb7d8331fe137d891fbdf13dfa98cb58be2d3",
          "seed": 1,
          "setup": "Granite 4.2 8B",
          "thinking": "off"
        },
        {
          "outcomes": "PPFFFFFFPFPPPPFFFF",
          "repetition": 2,
          "report_id": "79f5324a24db4cc6683f5865e00eb7d8331fe137d891fbdf13dfa98cb58be2d3",
          "seed": 2,
          "setup": "Granite 4.2 8B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPPPPFPPPPPPPPFPP",
          "repetition": 0,
          "report_id": "511dae62ca5252589cc84079f21e12e5a4e9da5859b8010ed9dba2db800e6cc2",
          "seed": 0,
          "setup": "Granite 4.2 8B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPFPPFPPPPPPFPPPP",
          "repetition": 1,
          "report_id": "511dae62ca5252589cc84079f21e12e5a4e9da5859b8010ed9dba2db800e6cc2",
          "seed": 1,
          "setup": "Granite 4.2 8B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPFPPPPPPPPFPP",
          "repetition": 2,
          "report_id": "511dae62ca5252589cc84079f21e12e5a4e9da5859b8010ed9dba2db800e6cc2",
          "seed": 2,
          "setup": "Granite 4.2 8B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPFPFPPPPPFFF",
          "repetition": 0,
          "report_id": "87114d424d910acc4d84092ad931c0475cb0d5065dd1225e549e670a2d5b032c",
          "seed": 0,
          "setup": "Gemma 4 12B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPPFPPPPFPPPPPFFF",
          "repetition": 1,
          "report_id": "87114d424d910acc4d84092ad931c0475cb0d5065dd1225e549e670a2d5b032c",
          "seed": 1,
          "setup": "Gemma 4 12B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPPPPPPPFPPPPPFFF",
          "repetition": 2,
          "report_id": "87114d424d910acc4d84092ad931c0475cb0d5065dd1225e549e670a2d5b032c",
          "seed": 2,
          "setup": "Gemma 4 12B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPPPPPPPPPPPPPPPP",
          "repetition": 0,
          "report_id": "e03a41c8c729e138a1fb4130837439a8db729ee21829c5ba0d89b80306493c30",
          "seed": 0,
          "setup": "Gemma 4 12B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPPPPPPPPPFPP",
          "repetition": 1,
          "report_id": "e03a41c8c729e138a1fb4130837439a8db729ee21829c5ba0d89b80306493c30",
          "seed": 1,
          "setup": "Gemma 4 12B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPPPPPPPPPPPP",
          "repetition": 2,
          "report_id": "e03a41c8c729e138a1fb4130837439a8db729ee21829c5ba0d89b80306493c30",
          "seed": 2,
          "setup": "Gemma 4 12B",
          "thinking": "on"
        },
        {
          "outcomes": "FPPPPPPFPPPPPFPPPF",
          "repetition": 0,
          "report_id": "9918f896f7234fe98b21795fb4937fb29b7dfe49b233ddc1bdda4c5881fb2c8e",
          "seed": 0,
          "setup": "LFM2.5-24B-A2B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPFPPPPPPPPPF",
          "repetition": 1,
          "report_id": "9918f896f7234fe98b21795fb4937fb29b7dfe49b233ddc1bdda4c5881fb2c8e",
          "seed": 1,
          "setup": "LFM2.5-24B-A2B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPPPPPPPPPPPF",
          "repetition": 2,
          "report_id": "9918f896f7234fe98b21795fb4937fb29b7dfe49b233ddc1bdda4c5881fb2c8e",
          "seed": 2,
          "setup": "LFM2.5-24B-A2B",
          "thinking": "on"
        },
        {
          "outcomes": "PPFPFFPPPFPPPPPFFP",
          "repetition": 0,
          "report_id": "33b2fc529fb2de88831d43e81b9afd563c90d7d8dbded7d334a2ac02081daff4",
          "seed": 0,
          "setup": "Qwen3.8-27B",
          "thinking": "off"
        },
        {
          "outcomes": "PPFFPPPPPFPPPPPFFF",
          "repetition": 1,
          "report_id": "33b2fc529fb2de88831d43e81b9afd563c90d7d8dbded7d334a2ac02081daff4",
          "seed": 1,
          "setup": "Qwen3.8-27B",
          "thinking": "off"
        },
        {
          "outcomes": "PPFPPFPPPFPPPPPFFP",
          "repetition": 2,
          "report_id": "33b2fc529fb2de88831d43e81b9afd563c90d7d8dbded7d334a2ac02081daff4",
          "seed": 2,
          "setup": "Qwen3.8-27B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPPPPFPPPPPPPPPPF",
          "repetition": 0,
          "report_id": "604dc3902cff19025cbfe96022094ed0376406a03797afe689cb8227ffe6786c",
          "seed": 0,
          "setup": "Qwen3.8-27B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPFPPPPPPPPPPF",
          "repetition": 1,
          "report_id": "604dc3902cff19025cbfe96022094ed0376406a03797afe689cb8227ffe6786c",
          "seed": 1,
          "setup": "Qwen3.8-27B",
          "thinking": "on"
        },
        {
          "outcomes": "PPFPPPPPPPPPPPPPPF",
          "repetition": 2,
          "report_id": "604dc3902cff19025cbfe96022094ed0376406a03797afe689cb8227ffe6786c",
          "seed": 2,
          "setup": "Qwen3.8-27B",
          "thinking": "on"
        },
        {
          "outcomes": "PPFFFPPFPFPPPPPFPF",
          "repetition": 0,
          "report_id": "f4b2e4a175cbb720ef525cacf7c1ce04ae09083a794457d2af61fbcf1f78e024",
          "seed": 0,
          "setup": "Gemma 4 31B",
          "thinking": "off"
        },
        {
          "outcomes": "PPFFFPPFPFPPPFPFPF",
          "repetition": 1,
          "report_id": "f4b2e4a175cbb720ef525cacf7c1ce04ae09083a794457d2af61fbcf1f78e024",
          "seed": 1,
          "setup": "Gemma 4 31B",
          "thinking": "off"
        },
        {
          "outcomes": "PPFFFPPPPFPPPPPFPF",
          "repetition": 2,
          "report_id": "f4b2e4a175cbb720ef525cacf7c1ce04ae09083a794457d2af61fbcf1f78e024",
          "seed": 2,
          "setup": "Gemma 4 31B",
          "thinking": "off"
        },
        {
          "outcomes": "PPPPPPPFPPPPPPPPPF",
          "repetition": 0,
          "report_id": "a92b0fc63f5e4d1d3c6ad852aca000c3a793ff08d84ee076b0ab1367a5930506",
          "seed": 0,
          "setup": "Gemma 4 31B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPFPPPPPPPPPF",
          "repetition": 1,
          "report_id": "a92b0fc63f5e4d1d3c6ad852aca000c3a793ff08d84ee076b0ab1367a5930506",
          "seed": 1,
          "setup": "Gemma 4 31B",
          "thinking": "on"
        },
        {
          "outcomes": "PPPPPPPFPPPPPPPPPF",
          "repetition": 2,
          "report_id": "a92b0fc63f5e4d1d3c6ad852aca000c3a793ff08d84ee076b0ab1367a5930506",
          "seed": 2,
          "setup": "Gemma 4 31B",
          "thinking": "on"
        }
      ],
      "task_ids": [
        "hard-long-chain-01-renamed",
        "hard-long-chain-02-renamed",
        "hard-long-chain-03-renamed",
        "hard-computed-args-01-renamed",
        "hard-computed-args-02-renamed",
        "hard-computed-args-03-renamed",
        "hard-computed-args-04-renamed",
        "hard-multi-constraint-03-renamed",
        "hard-multi-constraint-04-renamed",
        "hard-branching-01-renamed",
        "hard-branching-03-renamed",
        "hard-branching-04-renamed",
        "hard-cascading-recovery-01-renamed",
        "hard-cascading-recovery-02-renamed",
        "hard-cascading-recovery-03-renamed",
        "hard-long-output-01-renamed",
        "hard-long-output-02-renamed",
        "hard-long-output-04-renamed"
      ],
      "tasks": 18
    }
  }
}
