{
 "published": "2026-09-29",
 "label": "September 2026",
 "corpus_fingerprint": "f8636eff133b0699",
 "license": "CC-BY-4.0",
 "families": {
  "classification": {
   "label": "Document classification",
   "task_types": [
    "invoice.classification",
    "email.primary_classification",
    "support_ticket.category_classification"
   ]
  },
  "extraction": {
   "label": "Invoice total",
   "task_types": [
    "invoice.field_extraction"
   ]
  },
  "spreadsheet": {
   "label": "Spreadsheet reasoning",
   "task_types": [
    "spreadsheet.single_value_lookup",
    "spreadsheet.top_category_aggregation",
    "business.spreadsheet_operations"
   ]
  },
  "toolcall": {
   "label": "Tool calling",
   "task_types": [
    "toolcall.single",
    "toolcall.distractor"
   ]
  },
  "robustness": {
   "label": "Robustness and evidence",
   "task_types": [
    "business.document_robustness",
    "business.evidence_sufficiency",
    "business.robustness_confidence"
   ]
  }
 },
 "models": [
  {
   "rank": 1,
   "name": "Claude Opus 5.5",
   "maker": "Anthropic",
   "weights": "api",
   "size": "undisclosed",
   "quant": "Claude Agent SDK, effort medium",
   "runs_on": "Cloud API",
   "n": 216,
   "infra_excluded": 0,
   "passed": 199,
   "rate": 0.9212962962962963,
   "ci_low": 0.8775853063416398,
   "ci_high": 0.950283486461906,
   "failures": {
    "wrong_answer": 11,
    "malformed": 0,
    "truncated": 0,
    "context_overflow": 0,
    "timeout": 6,
    "refused": 0
   },
   "format_rate": 1.0,
   "format_ok": 210,
   "format_checked": 210,
   "median_latency_s": 2.7720000000000002,
   "cost_basis": "list_price_estimate",
   "cost_per_success_usd": 0.015883155778894473,
   "families": {
    "classification": 0.9259259259259259,
    "extraction": 1.0,
    "spreadsheet": 0.8518518518518519,
    "toolcall": 1.0,
    "robustness": 0.9074074074074074
   },
   "difficulty": {
    "1": 0.9722222222222222,
    "5": 0.9444444444444444,
    "9": 0.8472222222222222
   }
  },
  {
   "rank": 2,
   "name": "Claude Fable 5.1",
   "maker": "Anthropic",
   "weights": "api",
   "size": "undisclosed",
   "quant": "Claude Agent SDK, effort medium",
   "runs_on": "Cloud API",
   "n": 216,
   "infra_excluded": 0,
   "passed": 195,
   "rate": 0.9027777777777778,
   "ci_low": 0.8559491925043853,
   "ci_high": 0.9355297632528509,
   "failures": {
    "wrong_answer": 14,
    "malformed": 0,
    "truncated": 0,
    "context_overflow": 0,
    "timeout": 7,
    "refused": 0
   },
   "format_rate": 0.9569377990430622,
   "format_ok": 200,
   "format_checked": 209,
   "median_latency_s": 2.3,
   "cost_basis": "list_price_estimate",
   "cost_per_success_usd": 0.03449661538461538,
   "families": {
    "classification": 0.9074074074074074,
    "extraction": 1.0,
    "spreadsheet": 0.7407407407407407,
    "toolcall": 1.0,
    "robustness": 0.9629629629629629
   },
   "difficulty": {
    "1": 0.9722222222222222,
    "5": 0.9444444444444444,
    "9": 0.7916666666666666
   }
  },
  {
   "rank": 3,
   "name": "Claude Sonnet 5.5",
   "maker": "Anthropic",
   "weights": "api",
   "size": "undisclosed",
   "quant": "Claude Agent SDK, effort medium",
   "runs_on": "Cloud API",
   "n": 216,
   "infra_excluded": 0,
   "passed": 192,
   "rate": 0.8888888888888888,
   "ci_low": 0.8399976025091761,
   "ci_high": 0.9241889754633278,
   "failures": {
    "wrong_answer": 17,
    "malformed": 0,
    "truncated": 0,
    "context_overflow": 0,
    "timeout": 7,
    "refused": 0
   },
   "format_rate": 0.937799043062201,
   "format_ok": 196,
   "format_checked": 209,
   "median_latency_s": 1.839,
   "cost_basis": "list_price_estimate",
   "cost_per_success_usd": 0.007359947916666667,
   "families": {
    "classification": 0.8518518518518519,
    "extraction": 1.0,
    "spreadsheet": 0.7407407407407407,
    "toolcall": 1.0,
    "robustness": 0.9629629629629629
   },
   "difficulty": {
    "1": 0.9583333333333334,
    "5": 0.9444444444444444,
    "9": 0.7638888888888888
   }
  },
  {
   "rank": 4,
   "name": "DeepSeek V4.1 Flash",
   "maker": "DeepSeek",
   "weights": "api",
   "size": "undisclosed",
   "quant": "API (deepseek-flash)",
   "runs_on": "Cloud API",
   "n": 216,
   "infra_excluded": 0,
   "passed": 187,
   "rate": 0.8657407407407407,
   "ci_low": 0.8138308482210044,
   "ci_high": 0.9048684334436123,
   "failures": {
    "wrong_answer": 18,
    "malformed": 0,
    "truncated": 11,
    "context_overflow": 0,
    "timeout": 0,
    "refused": 0
   },
   "format_rate": 1.0,
   "format_ok": 205,
   "format_checked": 205,
   "median_latency_s": 1.38,
   "cost_basis": "recorded_plus_truncation_estimate",
   "cost_per_success_usd": 0.0023337160625257094,
   "families": {
    "classification": 0.8888888888888888,
    "extraction": 1.0,
    "spreadsheet": 0.6481481481481481,
    "toolcall": 1.0,
    "robustness": 0.9259259259259259
   },
   "difficulty": {
    "1": 0.9444444444444444,
    "5": 0.9305555555555556,
    "9": 0.7222222222222222
   }
  },
  {
   "rank": 5,
   "name": "DeepSeek V4 Pro (0813)",
   "maker": "DeepSeek",
   "weights": "api",
   "size": "undisclosed",
   "quant": "API (deepseek-v4-pro)",
   "runs_on": "Cloud API",
   "n": 216,
   "infra_excluded": 0,
   "passed": 185,
   "rate": 0.8564814814814815,
   "ci_low": 0.8034892255690972,
   "ci_high": 0.8970151375723648,
   "failures": {
    "wrong_answer": 20,
    "malformed": 0,
    "truncated": 11,
    "context_overflow": 0,
    "timeout": 0,
    "refused": 0
   },
   "format_rate": 1.0,
   "format_ok": 205,
   "format_checked": 205,
   "median_latency_s": 3.225,
   "cost_basis": "recorded_plus_truncation_estimate",
   "cost_per_success_usd": 0.008703168312681914,
   "families": {
    "classification": 0.8703703703703703,
    "extraction": 1.0,
    "spreadsheet": 0.6296296296296297,
    "toolcall": 1.0,
    "robustness": 0.9259259259259259
   },
   "difficulty": {
    "1": 0.9444444444444444,
    "5": 0.9027777777777778,
    "9": 0.7222222222222222
   }
  },
  {
   "rank": 6,
   "name": "gpt-oss-20b",
   "maker": "OpenAI",
   "weights": "open",
   "size": "21B MoE (3.6B active)",
   "quant": "MXFP4",
   "runs_on": "Local, 1\u00d7 16 GB GPU",
   "n": 216,
   "infra_excluded": 0,
   "passed": 171,
   "rate": 0.7916666666666666,
   "ci_low": 0.7326437318548641,
   "ci_high": 0.8404962016245138,
   "failures": {
    "wrong_answer": 23,
    "malformed": 10,
    "truncated": 12,
    "context_overflow": 0,
    "timeout": 0,
    "refused": 0
   },
   "format_rate": 0.9901477832512315,
   "format_ok": 201,
   "format_checked": 203,
   "median_latency_s": 3.479,
   "cost_basis": "local",
   "cost_per_success_usd": null,
   "families": {
    "classification": 0.8518518518518519,
    "extraction": 1.0,
    "spreadsheet": 0.5185185185185185,
    "toolcall": 1.0,
    "robustness": 0.7962962962962963
   },
   "difficulty": {
    "1": 0.9305555555555556,
    "5": 0.8055555555555556,
    "9": 0.6388888888888888
   }
  },
  {
   "rank": 7,
   "name": "Qwen3.8 27B (3-bit IQ3_XXS)",
   "maker": "Alibaba",
   "weights": "open",
   "size": "27B dense",
   "quant": "IQ3_XXS (GSQ-RCO, 10.1 GB)",
   "runs_on": "Local, 1\u00d7 16 GB GPU",
   "n": 216,
   "infra_excluded": 0,
   "passed": 151,
   "rate": 0.6990740740740741,
   "ci_low": 0.6348649672489957,
   "ci_high": 0.7563257809988337,
   "failures": {
    "wrong_answer": 65,
    "malformed": 0,
    "truncated": 0,
    "context_overflow": 0,
    "timeout": 0,
    "refused": 0
   },
   "format_rate": 1.0,
   "format_ok": 216,
   "format_checked": 216,
   "median_latency_s": 7.0115,
   "cost_basis": "local",
   "cost_per_success_usd": null,
   "families": {
    "classification": 0.8148148148148148,
    "extraction": 1.0,
    "spreadsheet": 0.07407407407407407,
    "toolcall": 1.0,
    "robustness": 0.9074074074074074
   },
   "difficulty": {
    "1": 0.75,
    "5": 0.6944444444444444,
    "9": 0.6527777777777778
   }
  },
  {
   "rank": 8,
   "name": "Gemma 4 26B-A4B (thinking)",
   "maker": "Google",
   "weights": "open",
   "size": "26B MoE (~4B active)",
   "quant": "Q4_K_M",
   "runs_on": "Local, 1\u00d7 16 GB GPU",
   "n": 216,
   "infra_excluded": 0,
   "passed": 150,
   "rate": 0.6944444444444444,
   "ci_low": 0.6300592551568772,
   "ci_high": 0.7520340338293746,
   "failures": {
    "wrong_answer": 60,
    "malformed": 2,
    "truncated": 0,
    "context_overflow": 0,
    "timeout": 4,
    "refused": 0
   },
   "format_rate": 0.9622641509433962,
   "format_ok": 204,
   "format_checked": 212,
   "median_latency_s": 34.4095,
   "cost_basis": "local",
   "cost_per_success_usd": null,
   "families": {
    "classification": 0.8518518518518519,
    "extraction": 1.0,
    "spreadsheet": 0.14814814814814814,
    "toolcall": 0.8333333333333334,
    "robustness": 0.8888888888888888
   },
   "difficulty": {
    "1": 0.7916666666666666,
    "5": 0.6805555555555556,
    "9": 0.6111111111111112
   }
  },
  {
   "rank": 9,
   "name": "Qwen3.8 27B",
   "maker": "Alibaba",
   "weights": "open",
   "size": "27B dense",
   "quant": "Q4_K_M (Unsloth UD)",
   "runs_on": "Local, 2\u00d7 16 GB GPU",
   "n": 216,
   "infra_excluded": 0,
   "passed": 149,
   "rate": 0.6898148148148148,
   "ci_low": 0.6252606303661626,
   "ci_high": 0.7477351993585118,
   "failures": {
    "wrong_answer": 67,
    "malformed": 0,
    "truncated": 0,
    "context_overflow": 0,
    "timeout": 0,
    "refused": 0
   },
   "format_rate": 1.0,
   "format_ok": 216,
   "format_checked": 216,
   "median_latency_s": 5.282,
   "cost_basis": "local",
   "cost_per_success_usd": null,
   "families": {
    "classification": 0.8148148148148148,
    "extraction": 1.0,
    "spreadsheet": 0.05555555555555555,
    "toolcall": 1.0,
    "robustness": 0.8888888888888888
   },
   "difficulty": {
    "1": 0.7361111111111112,
    "5": 0.6805555555555556,
    "9": 0.6527777777777778
   }
  },
  {
   "rank": 10,
   "name": "Gemma 4 12B",
   "maker": "Google",
   "weights": "open",
   "size": "12B dense",
   "quant": "Q4_0 (QAT)",
   "runs_on": "Local, 1\u00d7 16 GB GPU",
   "n": 216,
   "infra_excluded": 0,
   "passed": 148,
   "rate": 0.6851851851851852,
   "ci_low": 0.6204690064546781,
   "ci_high": 0.7434293640084191,
   "failures": {
    "wrong_answer": 68,
    "malformed": 0,
    "truncated": 0,
    "context_overflow": 0,
    "timeout": 0,
    "refused": 0
   },
   "format_rate": 0.9351851851851852,
   "format_ok": 202,
   "format_checked": 216,
   "median_latency_s": 2.951,
   "cost_basis": "local",
   "cost_per_success_usd": null,
   "families": {
    "classification": 0.8148148148148148,
    "extraction": 1.0,
    "spreadsheet": 0.05555555555555555,
    "toolcall": 0.9722222222222222,
    "robustness": 0.8888888888888888
   },
   "difficulty": {
    "1": 0.75,
    "5": 0.6805555555555556,
    "9": 0.625
   }
  },
  {
   "rank": 11,
   "name": "Qwen3.8 27B (3-bit IQ3_S)",
   "maker": "Alibaba",
   "weights": "open",
   "size": "27B dense",
   "quant": "IQ3_S (GSQ-RCO, 11.8 GB)",
   "runs_on": "Local, 1\u00d7 16 GB GPU",
   "n": 216,
   "infra_excluded": 0,
   "passed": 146,
   "rate": 0.6759259259259259,
   "ci_low": 0.6109064342276128,
   "ci_high": 0.7347970177123294,
   "failures": {
    "wrong_answer": 70,
    "malformed": 0,
    "truncated": 0,
    "context_overflow": 0,
    "timeout": 0,
    "refused": 0
   },
   "format_rate": 1.0,
   "format_ok": 216,
   "format_checked": 216,
   "median_latency_s": 7.293,
   "cost_basis": "local",
   "cost_per_success_usd": null,
   "families": {
    "classification": 0.7777777777777778,
    "extraction": 1.0,
    "spreadsheet": 0.037037037037037035,
    "toolcall": 1.0,
    "robustness": 0.8888888888888888
   },
   "difficulty": {
    "1": 0.7222222222222222,
    "5": 0.6666666666666666,
    "9": 0.6388888888888888
   }
  },
  {
   "rank": 12,
   "name": "Gemma 4 26B-A4B",
   "maker": "Google",
   "weights": "open",
   "size": "26B MoE (~4B active)",
   "quant": "Q4_K_M",
   "runs_on": "Local, 1\u00d7 16 GB GPU",
   "n": 216,
   "infra_excluded": 0,
   "passed": 142,
   "rate": 0.6574074074074074,
   "ci_low": 0.5918618957983307,
   "ci_high": 0.7174517190953019,
   "failures": {
    "wrong_answer": 72,
    "malformed": 2,
    "truncated": 0,
    "context_overflow": 0,
    "timeout": 0,
    "refused": 0
   },
   "format_rate": 1.0,
   "format_ok": 216,
   "format_checked": 216,
   "median_latency_s": 5.723,
   "cost_basis": "local",
   "cost_per_success_usd": null,
   "families": {
    "classification": 0.7592592592592593,
    "extraction": 1.0,
    "spreadsheet": 0.12962962962962962,
    "toolcall": 0.9722222222222222,
    "robustness": 0.7592592592592593
   },
   "difficulty": {
    "1": 0.6805555555555556,
    "5": 0.6944444444444444,
    "9": 0.5972222222222222
   }
  },
  {
   "rank": 13,
   "name": "Gemma 4 E4B",
   "maker": "Google",
   "weights": "open",
   "size": "~4B effective",
   "quant": "Q4_0",
   "runs_on": "Local, 1\u00d7 16 GB GPU",
   "n": 216,
   "infra_excluded": 0,
   "passed": 141,
   "rate": 0.6527777777777778,
   "ci_low": 0.5871171545984272,
   "ci_high": 0.7130990010336279,
   "failures": {
    "wrong_answer": 74,
    "malformed": 1,
    "truncated": 0,
    "context_overflow": 0,
    "timeout": 0,
    "refused": 0
   },
   "format_rate": 1.0,
   "format_ok": 216,
   "format_checked": 216,
   "median_latency_s": 1.4049999999999998,
   "cost_basis": "local",
   "cost_per_success_usd": null,
   "families": {
    "classification": 0.8333333333333334,
    "extraction": 1.0,
    "spreadsheet": 0.0,
    "toolcall": 1.0,
    "robustness": 0.7777777777777778
   },
   "difficulty": {
    "1": 0.7083333333333334,
    "5": 0.6666666666666666,
    "9": 0.5833333333333334
   }
  },
  {
   "rank": 14,
   "name": "Devstral Small 2 24B",
   "maker": "Mistral AI",
   "weights": "open",
   "size": "24B dense",
   "quant": "Q4_K_M",
   "runs_on": "Local, 2\u00d7 16 GB GPU",
   "n": 216,
   "infra_excluded": 0,
   "passed": 138,
   "rate": 0.6388888888888888,
   "ci_low": 0.5729212960893411,
   "ci_high": 0.7000024817579817,
   "failures": {
    "wrong_answer": 76,
    "malformed": 2,
    "truncated": 0,
    "context_overflow": 0,
    "timeout": 0,
    "refused": 0
   },
   "format_rate": 0.7453703703703703,
   "format_ok": 161,
   "format_checked": 216,
   "median_latency_s": 5.0095,
   "cost_basis": "local",
   "cost_per_success_usd": null,
   "families": {
    "classification": 0.7222222222222222,
    "extraction": 1.0,
    "spreadsheet": 0.0,
    "toolcall": 1.0,
    "robustness": 0.8333333333333334
   },
   "difficulty": {
    "1": 0.6805555555555556,
    "5": 0.6111111111111112,
    "9": 0.625
   }
  },
  {
   "rank": 15,
   "name": "Phi-4",
   "maker": "Microsoft",
   "weights": "open",
   "size": "14B dense",
   "quant": "Q4_K_M",
   "runs_on": "Local, 1\u00d7 16 GB GPU",
   "n": 216,
   "infra_excluded": 0,
   "passed": 135,
   "rate": 0.625,
   "ci_low": 0.5587818526716787,
   "ci_high": 0.6868495473909119,
   "failures": {
    "wrong_answer": 69,
    "malformed": 0,
    "truncated": 0,
    "context_overflow": 12,
    "timeout": 0,
    "refused": 0
   },
   "format_rate": 0.0,
   "format_ok": 0,
   "format_checked": 204,
   "median_latency_s": 3.124,
   "cost_basis": "local",
   "cost_per_success_usd": null,
   "families": {
    "classification": 0.7222222222222222,
    "extraction": 1.0,
    "spreadsheet": 0.0,
    "toolcall": 1.0,
    "robustness": 0.7777777777777778
   },
   "difficulty": {
    "1": 0.6944444444444444,
    "5": 0.6111111111111112,
    "9": 0.5694444444444444
   }
  },
  {
   "rank": 16,
   "name": "Qwen3-Coder 30B-A3B",
   "maker": "Alibaba",
   "weights": "open",
   "size": "30B MoE (3B active)",
   "quant": "Q4_K_M",
   "runs_on": "Local, 1\u00d7 16 GB GPU",
   "n": 216,
   "infra_excluded": 0,
   "passed": 134,
   "rate": 0.6203703703703703,
   "ci_low": 0.5540810124701837,
   "ci_high": 0.6824529283308293,
   "failures": {
    "wrong_answer": 81,
    "malformed": 0,
    "truncated": 0,
    "context_overflow": 0,
    "timeout": 1,
    "refused": 0
   },
   "format_rate": 0.986046511627907,
   "format_ok": 212,
   "format_checked": 215,
   "median_latency_s": 10.266,
   "cost_basis": "local",
   "cost_per_success_usd": null,
   "families": {
    "classification": 0.6666666666666666,
    "extraction": 1.0,
    "spreadsheet": 0.018518518518518517,
    "toolcall": 1.0,
    "robustness": 0.7962962962962963
   },
   "difficulty": {
    "1": 0.6944444444444444,
    "5": 0.5833333333333334,
    "9": 0.5833333333333334
   }
  },
  {
   "rank": 17,
   "name": "Mistral Small 3.2 24B",
   "maker": "Mistral AI",
   "weights": "open",
   "size": "24B dense",
   "quant": "Q4_K_M",
   "runs_on": "Local, 2\u00d7 16 GB GPU",
   "n": 216,
   "infra_excluded": 0,
   "passed": 133,
   "rate": 0.6157407407407407,
   "ci_low": 0.5493862518470368,
   "ci_high": 0.6780502296923988,
   "failures": {
    "wrong_answer": 79,
    "malformed": 4,
    "truncated": 0,
    "context_overflow": 0,
    "timeout": 0,
    "refused": 0
   },
   "format_rate": 0.7129629629629629,
   "format_ok": 154,
   "format_checked": 216,
   "median_latency_s": 5.2995,
   "cost_basis": "local",
   "cost_per_success_usd": null,
   "families": {
    "classification": 0.6851851851851852,
    "extraction": 1.0,
    "spreadsheet": 0.018518518518518517,
    "toolcall": 1.0,
    "robustness": 0.7592592592592593
   },
   "difficulty": {
    "1": 0.6666666666666666,
    "5": 0.5972222222222222,
    "9": 0.5833333333333334
   }
  },
  {
   "rank": 18,
   "name": "Qwen2.5-Coder 7B",
   "maker": "Alibaba",
   "weights": "open",
   "size": "7B dense",
   "quant": "Q4_K_M",
   "runs_on": "Local, 1\u00d7 16 GB GPU",
   "n": 216,
   "infra_excluded": 0,
   "passed": 132,
   "rate": 0.6111111111111112,
   "ci_low": 0.5446975294909372,
   "ci_high": 0.6736414927869211,
   "failures": {
    "wrong_answer": 82,
    "malformed": 2,
    "truncated": 0,
    "context_overflow": 0,
    "timeout": 0,
    "refused": 0
   },
   "format_rate": 0.9907407407407407,
   "format_ok": 214,
   "format_checked": 216,
   "median_latency_s": 1.7985,
   "cost_basis": "local",
   "cost_per_success_usd": null,
   "families": {
    "classification": 0.6666666666666666,
    "extraction": 1.0,
    "spreadsheet": 0.037037037037037035,
    "toolcall": 1.0,
    "robustness": 0.7407407407407407
   },
   "difficulty": {
    "1": 0.7222222222222222,
    "5": 0.5694444444444444,
    "9": 0.5416666666666666
   }
  },
  {
   "rank": 19,
   "name": "Qwen3 30B-A3B",
   "maker": "Alibaba",
   "weights": "open",
   "size": "30B MoE (3B active)",
   "quant": "Q4_K_M",
   "runs_on": "Local, 1\u00d7 16 GB GPU",
   "n": 216,
   "infra_excluded": 0,
   "passed": 130,
   "rate": 0.6018518518518519,
   "ci_low": 0.5353380445702955,
   "ci_high": 0.6648060591844078,
   "failures": {
    "wrong_answer": 86,
    "malformed": 0,
    "truncated": 0,
    "context_overflow": 0,
    "timeout": 0,
    "refused": 0
   },
   "format_rate": 1.0,
   "format_ok": 216,
   "format_checked": 216,
   "median_latency_s": 10.482,
   "cost_basis": "local",
   "cost_per_success_usd": null,
   "families": {
    "classification": 0.7222222222222222,
    "extraction": 1.0,
    "spreadsheet": 0.0,
    "toolcall": 1.0,
    "robustness": 0.6851851851851852
   },
   "difficulty": {
    "1": 0.6944444444444444,
    "5": 0.5555555555555556,
    "9": 0.5555555555555556
   }
  },
  {
   "rank": 20,
   "name": "DeepSeek-Coder-V2-Lite",
   "maker": "DeepSeek",
   "weights": "open",
   "size": "16B MoE (2.4B active)",
   "quant": "Q4_K_M",
   "runs_on": "Local, 1\u00d7 16 GB GPU",
   "n": 216,
   "infra_excluded": 0,
   "passed": 127,
   "rate": 0.5879629629629629,
   "ci_low": 0.5213431867545278,
   "ci_high": 0.6515085392154433,
   "failures": {
    "wrong_answer": 77,
    "malformed": 0,
    "truncated": 0,
    "context_overflow": 0,
    "timeout": 12,
    "refused": 0
   },
   "format_rate": 0.29411764705882354,
   "format_ok": 60,
   "format_checked": 204,
   "median_latency_s": 2.9530000000000003,
   "cost_basis": "local",
   "cost_per_success_usd": null,
   "families": {
    "classification": 0.6111111111111112,
    "extraction": 1.0,
    "spreadsheet": 0.05555555555555555,
    "toolcall": 1.0,
    "robustness": 0.6851851851851852
   },
   "difficulty": {
    "1": 0.7083333333333334,
    "5": 0.5138888888888888,
    "9": 0.5416666666666666
   }
  },
  {
   "rank": 21,
   "name": "Qwen3 8B",
   "maker": "Alibaba",
   "weights": "open",
   "size": "8B dense",
   "quant": "Q6_K",
   "runs_on": "Local, 1\u00d7 16 GB GPU",
   "n": 216,
   "infra_excluded": 0,
   "passed": 114,
   "rate": 0.5277777777777778,
   "ci_low": 0.4612972854109612,
   "ci_high": 0.5932874701585034,
   "failures": {
    "wrong_answer": 102,
    "malformed": 0,
    "truncated": 0,
    "context_overflow": 0,
    "timeout": 0,
    "refused": 0
   },
   "format_rate": 1.0,
   "format_ok": 216,
   "format_checked": 216,
   "median_latency_s": 2.2990000000000004,
   "cost_basis": "local",
   "cost_per_success_usd": null,
   "families": {
    "classification": 0.5185185185185185,
    "extraction": 1.0,
    "spreadsheet": 0.0,
    "toolcall": 1.0,
    "robustness": 0.5925925925925926
   },
   "difficulty": {
    "1": 0.625,
    "5": 0.4583333333333333,
    "9": 0.5
   }
  },
  {
   "rank": 22,
   "name": "Granite 8B Code Instruct",
   "maker": "IBM",
   "weights": "open",
   "size": "8B dense",
   "quant": "Q4_K_M",
   "runs_on": "Local, 1\u00d7 16 GB GPU",
   "n": 216,
   "infra_excluded": 0,
   "passed": 99,
   "rate": 0.4583333333333333,
   "ci_low": 0.39319192165446015,
   "ci_high": 0.5249309449913431,
   "failures": {
    "wrong_answer": 91,
    "malformed": 14,
    "truncated": 0,
    "context_overflow": 12,
    "timeout": 0,
    "refused": 0
   },
   "format_rate": 0.8725490196078431,
   "format_ok": 178,
   "format_checked": 204,
   "median_latency_s": 2.3095,
   "cost_basis": "local",
   "cost_per_success_usd": null,
   "families": {
    "classification": 0.3148148148148148,
    "extraction": 1.0,
    "spreadsheet": 0.0,
    "toolcall": 1.0,
    "robustness": 0.5185185185185185
   },
   "difficulty": {
    "1": 0.5277777777777778,
    "5": 0.375,
    "9": 0.4722222222222222
   }
  }
 ]
}