{
  "bench": "tool-calling",
  "models": [
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 1.7B",
      "commit": "0dc4268b561499c648c7ba7c9466bf3d8d01517a",
      "display_name": "LFM3-1.7B",
      "gpu_cost_usd": 0.00664,
      "gpu_hours": 0.0026666666666666666,
      "improvement_points": null,
      "kind": "chat",
      "maker": "Liquid AI",
      "model_key": "toolbench-5e65142ae9cfa7bb",
      "name": "LFM3-1.7B",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#9528)",
      "params_active": 1.73,
      "params_total": 1.73,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.8452380952380952
      },
      "repo": "LiquidAI/LFM3-1.7B",
      "runs": [
        "report:5e65142ae9cfa7bb6cf0ca3c53bd63a502a255d13ee66aea80faeb782411f5e6"
      ],
      "score_0_100": 84.52380952380952,
      "setup_label": "off",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 1.7B",
      "commit": "0dc4268b561499c648c7ba7c9466bf3d8d01517a",
      "display_name": "LFM3-1.7B",
      "gpu_cost_usd": 0.016184999999999998,
      "gpu_hours": 0.0065,
      "improvement_points": null,
      "kind": "chat",
      "maker": "Liquid AI",
      "model_key": "toolbench-27023f16baa4af5f",
      "name": "LFM3-1.7B",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#9528)",
      "params_active": 1.73,
      "params_total": 1.73,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.9166666666666666
      },
      "repo": "LiquidAI/LFM3-1.7B",
      "runs": [
        "report:27023f16baa4af5f5d659c2e14bc79406f064a356825f8e5ca094385cd6cd726"
      ],
      "score_0_100": 91.66666666666666,
      "setup_label": "on",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 5.1B",
      "commit": "4adf15748de9e1a0097b6274cee9c15836c41c37",
      "display_name": "Gemma 4 E2B",
      "gpu_cost_usd": 0.04924666666666667,
      "gpu_hours": 0.01977777777777778,
      "improvement_points": null,
      "kind": "chat",
      "maker": "Google",
      "model_key": "toolbench-140f843536d8fa04",
      "name": "Gemma 4 E2B",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#9534)",
      "params_active": 5.1,
      "params_total": 5.1,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.9404761904761905
      },
      "repo": "google/gemma-4-E2B-it",
      "runs": [
        "report:140f843536d8fa04928edaa53d3bc8ad5a9e8e0d5fe47b5a7ee4345377f94fbf"
      ],
      "score_0_100": 94.04761904761905,
      "setup_label": "on",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 2.5B",
      "commit": "42433e90dad95123ab87729fd7011e7110629556",
      "display_name": "MiniCPM5-2B",
      "gpu_cost_usd": 0.008507500000000001,
      "gpu_hours": 0.003416666666666667,
      "improvement_points": null,
      "kind": "chat",
      "maker": "OpenBMB",
      "model_key": "toolbench-803d110ff1b44abe",
      "name": "MiniCPM5-2B",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#9641)",
      "params_active": 2.52,
      "params_total": 2.52,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.8134920634920635
      },
      "repo": "openbmb/MiniCPM5-2B",
      "runs": [
        "report:803d110ff1b44abe87c974a24937c1e8d4093cb87d9b07269cac4423bfcc9877"
      ],
      "score_0_100": 81.34920634920636,
      "setup_label": "off",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 2.5B",
      "commit": "42433e90dad95123ab87729fd7011e7110629556",
      "display_name": "MiniCPM5-2B",
      "gpu_cost_usd": 0.03970166666666667,
      "gpu_hours": 0.015944444444444445,
      "improvement_points": null,
      "kind": "chat",
      "maker": "OpenBMB",
      "model_key": "toolbench-31def253c1044142",
      "name": "MiniCPM5-2B",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#9641)",
      "params_active": 2.52,
      "params_total": 2.52,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.9047619047619048
      },
      "repo": "openbmb/MiniCPM5-2B",
      "runs": [
        "report:31def253c10441426193b6fb7fc4556defb2faceda280e9dd5a39c278812a4e0"
      ],
      "score_0_100": 90.47619047619048,
      "setup_label": "on",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 2.0B",
      "commit": "15852e8c16360a2fea060d615a32b45270f8a8fc",
      "display_name": "Qwen3.5-2B",
      "gpu_cost_usd": 0.017983333333333337,
      "gpu_hours": 0.007222222222222222,
      "improvement_points": null,
      "kind": "chat",
      "maker": "Alibaba",
      "model_key": "toolbench-c6844a36b0fb5e06",
      "name": "Qwen3.5-2B",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#9668)",
      "params_active": 2.0,
      "params_total": 2.0,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.7341269841269841
      },
      "repo": "Qwen/Qwen3.5-2B",
      "runs": [
        "report:c6844a36b0fb5e0618609d3a0fe44a7269dce3e9a31ef2982b4fd2a306a98865"
      ],
      "score_0_100": 73.4126984126984,
      "setup_label": "off",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 2.0B",
      "commit": "15852e8c16360a2fea060d615a32b45270f8a8fc",
      "display_name": "Qwen3.5-2B",
      "gpu_cost_usd": 0.041085,
      "gpu_hours": 0.0165,
      "improvement_points": null,
      "kind": "chat",
      "maker": "Alibaba",
      "model_key": "toolbench-2de7e756ad1fec9e",
      "name": "Qwen3.5-2B",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#9668)",
      "params_active": 2.0,
      "params_total": 2.0,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.873015873015873
      },
      "repo": "Qwen/Qwen3.5-2B",
      "runs": [
        "report:2de7e756ad1fec9ec378454dd1cce2212bcb990ce5c31ae28e0c212242f22745"
      ],
      "score_0_100": 87.3015873015873,
      "setup_label": "on",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 4.7B",
      "commit": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",
      "display_name": "Qwen3.5 4B",
      "gpu_cost_usd": 0.021580000000000002,
      "gpu_hours": 0.008666666666666666,
      "improvement_points": null,
      "kind": "chat",
      "maker": "Alibaba",
      "model_key": "toolbench-1340618e38e52461",
      "name": "Qwen3.5 4B",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#9689)",
      "params_active": 4.66,
      "params_total": 4.66,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.9126984126984127
      },
      "repo": "Qwen/Qwen3.5-4B",
      "runs": [
        "report:1340618e38e52461d9ddb8049f2e27e8cbdbcb5000a4342d6aeab84cc52f7090"
      ],
      "score_0_100": 91.26984126984127,
      "setup_label": "off",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 4.7B",
      "commit": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a",
      "display_name": "Qwen3.5 4B",
      "gpu_cost_usd": 0.05388083333333334,
      "gpu_hours": 0.02163888888888889,
      "improvement_points": null,
      "kind": "chat",
      "maker": "Alibaba",
      "model_key": "toolbench-b58076261a85080e",
      "name": "Qwen3.5 4B",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#9689)",
      "params_active": 4.66,
      "params_total": 4.66,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.8968253968253969
      },
      "repo": "Qwen/Qwen3.5-4B",
      "runs": [
        "report:b58076261a85080eabc0734a1ab45639ec8692f4b5b04d45a7668f1fdba0a5f3"
      ],
      "score_0_100": 89.68253968253968,
      "setup_label": "on",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 3.1B",
      "commit": "fcf2d1d55e84e2f9fbe7b5d724dcfc4d861cfcee",
      "display_name": "Raon-1 nano",
      "gpu_cost_usd": 0.036865833333333334,
      "gpu_hours": 0.014805555555555554,
      "improvement_points": null,
      "kind": "chat",
      "maker": "KRAFTON",
      "model_key": "toolbench-26cc2deb86a2db78",
      "name": "Raon-1 nano",
      "notes": "original/renamed paired, no chance correction; provider local KRAFTON vLLM fork served by @logic on bariton (#9725)",
      "params_active": 3.1,
      "params_total": 3.1,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.7976190476190477
      },
      "repo": "KRAFTON/Raon-1-nano",
      "runs": [
        "report:26cc2deb86a2db785bd7976cd409d9f96b076832f8cb997d987984747508774c"
      ],
      "score_0_100": 79.76190476190477,
      "setup_label": "off",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 3.1B",
      "commit": "fcf2d1d55e84e2f9fbe7b5d724dcfc4d861cfcee",
      "display_name": "Raon-1 nano",
      "gpu_cost_usd": 0.06003666666666667,
      "gpu_hours": 0.02411111111111111,
      "improvement_points": null,
      "kind": "chat",
      "maker": "KRAFTON",
      "model_key": "toolbench-c52d4f9608a3fb8f",
      "name": "Raon-1 nano",
      "notes": "original/renamed paired, no chance correction; provider local KRAFTON vLLM fork served by @logic on bariton (#9725)",
      "params_active": 3.1,
      "params_total": 3.1,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.8888888888888888
      },
      "repo": "KRAFTON/Raon-1-nano",
      "runs": [
        "report:c52d4f9608a3fb8f97a39d8f339f2e7d41b9da73cbe8b097af75329687eea5be"
      ],
      "score_0_100": 88.88888888888889,
      "setup_label": "on",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 5.1B",
      "commit": "4adf15748de9e1a0097b6274cee9c15836c41c37",
      "display_name": "Gemma 4 E2B (tool-result cue template)",
      "gpu_cost_usd": 0.021580000000000002,
      "gpu_hours": 0.008666666666666666,
      "improvement_points": null,
      "kind": "chat",
      "maker": "Google",
      "model_key": "toolbench-19a04105741d831b",
      "name": "Gemma 4 E2B (tool-result cue template)",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#10207 GPU 3 for qualification; #10260 GPU 2 for the run, identical argv/image/template); Non-maker template (\u00dcmit #10181): the maker's chat template plus the two-line fix proposed in https://github.com/google-deepmind/gemma/issues/829 (also added by Google to gemma-4-31B-it at commit ad62905): an empty thought '<|channel>thought\\n<channel|>' after a tool result with thinking off (template sha256 84d5a64c\u2026). Thinking is off by request, but the parser classed 58% of output tokens as reasoning (275 of 580 replies). The maker-template thinking-off setup stays 'Not qualified'.",
      "params_active": 5.1,
      "params_total": 5.1,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.8928571428571429
      },
      "repo": "google/gemma-4-E2B-it",
      "runs": [
        "report:19a04105741d831b532081e524955585f9c6635d5a1d93b10080adcf991a5496"
      ],
      "score_0_100": 89.28571428571429,
      "setup_label": "off",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 3.0B",
      "commit": "4e9c275e232f10211ec88644d106a1f6a1b60e06",
      "display_name": "Granite 4.2 3B",
      "gpu_cost_usd": 0.024900000000000005,
      "gpu_hours": 0.01,
      "improvement_points": null,
      "kind": "chat",
      "maker": "IBM",
      "model_key": "toolbench-315eb4775b3311eb",
      "name": "Granite 4.2 3B",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#10256)",
      "params_active": 3.0,
      "params_total": 3.0,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.7936507936507936
      },
      "repo": "ibm-granite/granite-4.2-3b",
      "runs": [
        "report:315eb4775b3311ebb6fe55b312f82a70df77904742fc3047edb4c1d22405b167"
      ],
      "score_0_100": 79.36507936507937,
      "setup_label": "off",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 3.0B",
      "commit": "4e9c275e232f10211ec88644d106a1f6a1b60e06",
      "display_name": "Granite 4.2 3B",
      "gpu_cost_usd": 0.10886833333333334,
      "gpu_hours": 0.043722222222222225,
      "improvement_points": null,
      "kind": "chat",
      "maker": "IBM",
      "model_key": "toolbench-200b3adfe6fc2633",
      "name": "Granite 4.2 3B",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#10256)",
      "params_active": 3.0,
      "params_total": 3.0,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.9285714285714286
      },
      "repo": "ibm-granite/granite-4.2-3b",
      "runs": [
        "report:200b3adfe6fc263356bd06c28bb143cedecbb9c877db8b46a682e51c893b0471"
      ],
      "score_0_100": 92.85714285714286,
      "setup_label": "on",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 8.0B",
      "commit": "a7042144125ce5ae64f89610e8289647fc000b9a",
      "display_name": "Granite 4.2 8B",
      "gpu_cost_usd": 0.05644,
      "gpu_hours": 0.022666666666666665,
      "improvement_points": null,
      "kind": "chat",
      "maker": "IBM",
      "model_key": "toolbench-71d9250a150d74bd",
      "name": "Granite 4.2 8B",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#10281)",
      "params_active": 8.0,
      "params_total": 8.0,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.8373015873015873
      },
      "repo": "ibm-granite/granite-4.2-8b",
      "runs": [
        "report:71d9250a150d74bd08882aef5d9c6cb9c0b6c7faa795b471b724d5110e895242"
      ],
      "score_0_100": 83.73015873015873,
      "setup_label": "off",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 8.0B",
      "commit": "a7042144125ce5ae64f89610e8289647fc000b9a",
      "display_name": "Granite 4.2 8B",
      "gpu_cost_usd": 0.22935666666666668,
      "gpu_hours": 0.09211111111111112,
      "improvement_points": null,
      "kind": "chat",
      "maker": "IBM",
      "model_key": "toolbench-3ed26ffbcd602b69",
      "name": "Granite 4.2 8B",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#10281)",
      "params_active": 8.0,
      "params_total": 8.0,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.9325396825396826
      },
      "repo": "ibm-granite/granite-4.2-8b",
      "runs": [
        "report:3ed26ffbcd602b69bf2bdf8e3931eb85fe5e7d8fe3b86e072c51576497ba4b99"
      ],
      "score_0_100": 93.25396825396825,
      "setup_label": "on",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 12B",
      "commit": "43c9d014b231474fa7212dbb48a762566c2b255d",
      "display_name": "Gemma 4 12B",
      "gpu_cost_usd": 0.02656,
      "gpu_hours": 0.010666666666666666,
      "improvement_points": null,
      "kind": "chat",
      "maker": "Google",
      "model_key": "toolbench-3a310919357f8495",
      "name": "Gemma 4 12B",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#10304)",
      "params_active": 11.95,
      "params_total": 11.95,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.9444444444444444
      },
      "repo": "google/gemma-4-12B-it",
      "runs": [
        "report:3a310919357f8495163596163db17dbf37d8243f612a8fee8ad639b0b04d38cb"
      ],
      "score_0_100": 94.44444444444444,
      "setup_label": "off",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 12B",
      "commit": "43c9d014b231474fa7212dbb48a762566c2b255d",
      "display_name": "Gemma 4 12B",
      "gpu_cost_usd": 0.08355333333333334,
      "gpu_hours": 0.033555555555555554,
      "improvement_points": null,
      "kind": "chat",
      "maker": "Google",
      "model_key": "toolbench-449350778855864d",
      "name": "Gemma 4 12B",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#10304)",
      "params_active": 11.95,
      "params_total": 11.95,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.9285714285714286
      },
      "repo": "google/gemma-4-12B-it",
      "runs": [
        "report:449350778855864d3c4bb9d94c9c09aea997425277dc1193015f6c3c93f3922a"
      ],
      "score_0_100": 92.85714285714286,
      "setup_label": "on",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 24B MoE \u00b7 2.5B active",
      "commit": "1769516ac588c4c558089c94f27d96b0dc67b183",
      "display_name": "LFM2.5-24B-A2B",
      "gpu_cost_usd": 0.09323666666666668,
      "gpu_hours": 0.03744444444444445,
      "improvement_points": null,
      "kind": "chat",
      "maker": "Liquid AI",
      "model_key": "toolbench-ab8b10f2c435f634",
      "name": "LFM2.5-24B-A2B",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#10154)",
      "params_active": 2.45,
      "params_total": 24.0,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.9285714285714286
      },
      "repo": "LiquidAI/LFM2.5-24B-A2B",
      "runs": [
        "report:ab8b10f2c435f6346f219b9ec2434c2931462f4794e9a3a3e62c6fea993afbaa"
      ],
      "score_0_100": 92.85714285714286,
      "setup_label": "on",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 27B",
      "commit": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
      "display_name": "Qwen3.8-27B",
      "gpu_cost_usd": 0.09932333333333333,
      "gpu_hours": 0.03988888888888889,
      "improvement_points": null,
      "kind": "chat",
      "maker": "Alibaba",
      "model_key": "toolbench-540de94d654c683b",
      "name": "Qwen3.8-27B",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#10154)",
      "params_active": 27.0,
      "params_total": 27.0,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.9682539682539683
      },
      "repo": "Qwen/Qwen3.8-27B",
      "runs": [
        "report:540de94d654c683bd368da1a75c12428a9651864c783fecb2d6db1efb25c5710"
      ],
      "score_0_100": 96.82539682539682,
      "setup_label": "off",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 27B",
      "commit": "1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0",
      "display_name": "Qwen3.8-27B",
      "gpu_cost_usd": 0.2176675,
      "gpu_hours": 0.08741666666666667,
      "improvement_points": null,
      "kind": "chat",
      "maker": "Alibaba",
      "model_key": "toolbench-d072f54b34ce4ea0",
      "name": "Qwen3.8-27B",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#10154)",
      "params_active": 27.0,
      "params_total": 27.0,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.9206349206349206
      },
      "repo": "Qwen/Qwen3.8-27B",
      "runs": [
        "report:d072f54b34ce4ea09e6e8023f3129093965fe4ec2cb27cedccd46acb73359aaf"
      ],
      "score_0_100": 92.06349206349206,
      "setup_label": "on",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 31B",
      "commit": "842da3794eaa0b77d5f08bae87a17459d91ff475",
      "display_name": "Gemma 4 31B",
      "gpu_cost_usd": 0.05941416666666667,
      "gpu_hours": 0.023861111111111114,
      "improvement_points": null,
      "kind": "chat",
      "maker": "Google",
      "model_key": "toolbench-55d109bbc1ec0970",
      "name": "Gemma 4 31B",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#10154 GPU 0 for qualification; #10267 GPU 2 for the runs, argv/image/template identical); At the ceiling: 98% or more on both tracks (Toolbench's near-ceiling rule), so this task set cannot tell it apart from a perfect score; harder tasks would be needed to rank it further.",
      "params_active": 30.7,
      "params_total": 30.7,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 1.0
      },
      "repo": "google/gemma-4-31B-it",
      "runs": [
        "report:55d109bbc1ec0970e88c9c1c787786a7c2e4417ac93360986f37abd225615559"
      ],
      "score_0_100": 100.0,
      "setup_label": "off",
      "tag": "chat"
    },
    {
      "base_model": null,
      "calibration_error": null,
      "class_label": "Open \u00b7 31B",
      "commit": "842da3794eaa0b77d5f08bae87a17459d91ff475",
      "display_name": "Gemma 4 31B",
      "gpu_cost_usd": 0.17402333333333334,
      "gpu_hours": 0.06988888888888889,
      "improvement_points": null,
      "kind": "chat",
      "maker": "Google",
      "model_key": "toolbench-23e74f4bd54a0e12",
      "name": "Gemma 4 31B",
      "notes": "original/renamed paired, no chance correction; provider local vLLM served by @logic on bariton (#10154 GPU 0 for qualification; #10267 GPU 2 for the runs, argv/image/template identical); At the ceiling: 98% or more on both tracks (Toolbench's near-ceiling rule), so this task set cannot tell it apart from a perfect score; harder tasks would be needed to rank it further.",
      "params_active": 30.7,
      "params_total": 30.7,
      "quantization": null,
      "raw": {
        "attempts": 252,
        "metric": "task_pass_rate",
        "repetitions": 3,
        "tasks_per_track": 42,
        "tracks": 2,
        "value": 0.9880952380952381
      },
      "repo": "google/gemma-4-31B-it",
      "runs": [
        "report:23e74f4bd54a0e12636cd523d94479ad9fff48615c4bbed2e09b6d4a7d3a8cee"
      ],
      "score_0_100": 98.80952380952381,
      "setup_label": "on",
      "tag": "chat"
    }
  ],
  "score": {
    "hundred": "Every task passed",
    "zero": "No tasks passed"
  },
  "title": "Can a small model use tools?",
  "updated": "2026-11-16T18:29:23+01:00",
  "url": "https://zekiworks.github.io/analysis/tool-calling/"
}
