{
 "models": {
  "claude-sonnet-5-5-high": "Claude Sonnet 5.5",
  "claude-opus-5-5-high": "Claude Opus 5.5",
  "gemini-3.8-flash-high": "Gemini 3.8 Flash",
  "gemini-3.1-flash-lite": "Gemini 3.1 Flash-Lite",
  "gemini-4-argon-high": "Gemini 4 Argon",
  "gpt-oss-120b-medium": "gpt-oss-120b",
  "claude-haiku-5-5": "Claude Haiku 5.5",
  "claude-fable-5-1": "Claude Fable 5.1",
  "codex:gpt-6.1-sol": "GPT-6.1 Sol",
  "codex:gpt-6.1-terra": "GPT-6.1 Terra",
  "codex:gpt-6-luna": "GPT-6 Luna",
  "codex:gpt-5.6-sol": "GPT-5.6 Sol",
  "claude:claude-fable-5-2": "Claude Fable 5.2"
 },
 "prices": {
  "claude-sonnet-5-5-high": {
   "price_model": "Claude Sonnet 5.5",
   "input": 2.0,
   "cache_write": 2.5,
   "cached_input": 0.1,
   "output": 10.0,
   "url": "https://platform.claude.com/docs/en/about-claude/pricing",
   "note": "5m cache write $2.50; cache hit $0.10"
  },
  "claude-opus-5-5-high": {
   "price_model": "Claude Opus 5.5",
   "input": 4.0,
   "cache_write": 5.0,
   "cached_input": 0.2,
   "output": 20.0,
   "url": "https://platform.claude.com/docs/en/about-claude/pricing",
   "note": "5m cache write $5; cache hit $0.20"
  },
  "claude-haiku-5-5": {
   "price_model": "Claude Haiku 5.5 (prompts up to 100k tokens)",
   "input": 0.1,
   "cache_write": 0.125,
   "cached_input": 0.01,
   "output": 0.5,
   "url": "https://platform.claude.com/docs/en/about-claude/pricing",
   "note": "served natively on the proxy, measured usage"
  },
  "claude-fable-5-1": {
   "price_model": "Claude Fable 5.1",
   "input": 10.0,
   "cache_write": 12.5,
   "cached_input": 0.25,
   "output": 50.0,
   "url": "https://platform.claude.com/docs/en/about-claude/pricing",
   "note": "read 2026-11-12; served natively on the proxy, measured usage"
  },
  "gemini-3.8-flash-high": {
   "price_model": "Gemini 3.8 Flash",
   "input": 0.75,
   "cache_write": null,
   "cached_input": 0.075,
   "output": 3.75,
   "url": "https://aistudio.google.com/docs/pricing",
   "note": "Price doubles 2027-01-01 (input 1.50, output 7.50). Output includes thinking tokens. Cache storage billed per hour (not modelled)."
  },
  "gemini-3.1-flash-lite": {
   "price_model": "Gemini 3.1 Flash-Lite",
   "input": 0.25,
   "cache_write": null,
   "cached_input": 0.025,
   "output": 1.5,
   "url": "https://aistudio.google.com/docs/pricing",
   "note": "text input price"
  },
  "gemini-4-argon-high": {
   "price_model": "Gemini 4 Argon",
   "input": 2.0,
   "cache_write": null,
   "cached_input": 0.1,
   "output": 10.0,
   "url": "https://aistudio.google.com/docs/pricing",
   "note": "Listed on the same page; price through 2026-11-30, doubles 2026-12-01 (input 4.00, output 20.00). Implicit caching only."
  },
  "gpt-oss-120b-medium": {
   "price_model": "gpt-oss-120b on Groq",
   "host": "Groq",
   "input": 0.15,
   "cache_write": null,
   "cached_input": 0.075,
   "output": 0.6,
   "url": "https://console.groq.com/docs/model/openai/gpt-oss-120b",
   "note": "Open-weight model; price varies by host. Groq used as the single reference. The proxy's real backend host is unknown."
  },
  "codex:gpt-6.1-sol": {
   "price_model": "GPT-6.1 Sol",
   "input": 2.0,
   "cached_input": 0.1,
   "cache_write": null,
   "output": 10.0,
   "url": "https://developers.openai.com/api/docs/pricing",
   "note": "list price x ESTIMATED tokens (tiktoken o200k_base on input text + reply text); CLI harness token counts not used. openai.com/api/pricing returned HTTP 403 to our fetcher; developers.openai.com is OpenAI's own docs pricing page."
  },
  "codex:gpt-6.1-terra": {
   "price_model": "GPT-6.1 Terra",
   "input": 1.0,
   "cached_input": 0.05,
   "cache_write": null,
   "output": 5.0,
   "url": "https://developers.openai.com/api/docs/pricing",
   "note": "list price x ESTIMATED tokens (tiktoken o200k_base on input text + reply text); CLI harness token counts not used. openai.com/api/pricing returned HTTP 403 to our fetcher; developers.openai.com is OpenAI's own docs pricing page."
  },
  "codex:gpt-6-luna": {
   "price_model": "GPT-6 Luna",
   "input": 0.1,
   "cached_input": 0.01,
   "cache_write": null,
   "output": 0.5,
   "url": "https://developers.openai.com/api/docs/pricing",
   "note": "list price x ESTIMATED tokens (tiktoken o200k_base on input text + reply text); CLI harness token counts not used. openai.com/api/pricing returned HTTP 403 to our fetcher; developers.openai.com is OpenAI's own docs pricing page."
  },
  "codex:gpt-5.6-sol": {
   "price_model": "GPT-5.6 Sol (promo pricing noted on page until 2026-11-30)",
   "input": 4.0,
   "cached_input": 0.4,
   "cache_write": null,
   "output": 20.0,
   "url": "https://developers.openai.com/api/docs/pricing",
   "note": "list price x ESTIMATED tokens (tiktoken o200k_base on input text + reply text); CLI harness token counts not used. openai.com/api/pricing returned HTTP 403 to our fetcher; developers.openai.com is OpenAI's own docs pricing page."
  },
  "claude:claude-fable-5-2": {
   "price_model": "Claude Fable 5.2",
   "input": 8.0,
   "cached_input": 0.2,
   "cache_write": 10.0,
   "output": 40.0,
   "url": "https://platform.claude.com/docs/en/about-claude/pricing",
   "note": "list price x ESTIMATED tokens (tiktoken o200k_base on input text + reply text); CLI harness token counts not used. openai.com/api/pricing returned HTTP 403 to our fetcher; developers.openai.com is OpenAI's own docs pricing page."
  }
 },
 "errors_excluded": {},
 "counts": {
  "Claude Sonnet 5.5": {
   "support": 80,
   "tools": 80,
   "extract": 80
  },
  "Claude Opus 5.5": {
   "support": 80,
   "tools": 80,
   "extract": 80
  },
  "Gemini 3.8 Flash": {
   "support": 80,
   "tools": 80,
   "extract": 80
  },
  "Gemini 3.1 Flash-Lite": {
   "support": 80,
   "tools": 80,
   "extract": 80
  },
  "Gemini 4 Argon": {
   "support": 80,
   "tools": 80,
   "extract": 80
  },
  "gpt-oss-120b": {
   "support": 80,
   "tools": 80,
   "extract": 80
  },
  "Claude Haiku 5.5": {
   "support": 80,
   "tools": 80,
   "extract": 80
  },
  "Claude Fable 5.1": {
   "support": 80,
   "tools": 80,
   "extract": 80
  },
  "GPT-6.1 Sol": {},
  "GPT-6.1 Terra": {},
  "GPT-6 Luna": {},
  "GPT-5.6 Sol": {},
  "Claude Fable 5.2": {
   "support": 80,
   "extract": 80
  }
 },
 "table_paired_subset_support_extract_all_models": {
  "claude-sonnet-5-5-high": {
   "label": "Claude Sonnet 5.5",
   "n": 160,
   "correct": 156,
   "accuracy": 0.975,
   "accuracy_ci95": [
    0.9375,
    0.9902
   ],
   "usd_per_1k_answers": 5.1198250000000005,
   "usd_per_correct": 0.005251102564102564,
   "avg_prompt_tokens": 1277.81875,
   "avg_completion_tokens": 256.41875,
   "avg_reasoning_tokens": 0,
   "token_basis": "measured",
   "latency_median_s": 2.755
  },
  "claude-opus-5-5-high": {
   "label": "Claude Opus 5.5",
   "n": 160,
   "correct": 155,
   "accuracy": 0.96875,
   "accuracy_ci95": [
    0.9289,
    0.9866
   ],
   "usd_per_1k_answers": 11.589775,
   "usd_per_correct": 0.011963638709677418,
   "avg_prompt_tokens": 1277.81875,
   "avg_completion_tokens": 323.925,
   "avg_reasoning_tokens": 0,
   "token_basis": "measured",
   "latency_median_s": 3.24
  },
  "gemini-3.8-flash-high": {
   "label": "Gemini 3.8 Flash",
   "n": 160,
   "correct": 156,
   "accuracy": 0.975,
   "accuracy_ci95": [
    0.9375,
    0.9902
   ],
   "usd_per_1k_answers": 3.6236718750000003,
   "usd_per_correct": 0.0037165865384615384,
   "avg_prompt_tokens": 948.71875,
   "avg_completion_tokens": 776.56875,
   "avg_reasoning_tokens": 653.375,
   "token_basis": "measured",
   "latency_median_s": 3.26
  },
  "gemini-3.1-flash-lite": {
   "label": "Gemini 3.1 Flash-Lite",
   "n": 160,
   "correct": 144,
   "accuracy": 0.9,
   "accuracy_ci95": [
    0.8437,
    0.9375
   ],
   "usd_per_1k_answers": 0.4047109375,
   "usd_per_correct": 0.00044967881944444444,
   "avg_prompt_tokens": 948.71875,
   "avg_completion_tokens": 111.6875,
   "avg_reasoning_tokens": 0,
   "token_basis": "measured",
   "latency_median_s": 0.99
  },
  "gemini-4-argon-high": {
   "label": "Gemini 4 Argon",
   "n": 160,
   "correct": 154,
   "accuracy": 0.9625,
   "accuracy_ci95": [
    0.9206,
    0.9827
   ],
   "usd_per_1k_answers": 3.9231500000000006,
   "usd_per_correct": 0.004076000000000001,
   "avg_prompt_tokens": 978.325,
   "avg_completion_tokens": 196.65,
   "avg_reasoning_tokens": 55.275,
   "token_basis": "measured",
   "latency_median_s": 7.295
  },
  "gpt-oss-120b-medium": {
   "label": "gpt-oss-120b",
   "n": 160,
   "correct": 147,
   "accuracy": 0.91875,
   "accuracy_ci95": [
    0.866,
    0.9519
   ],
   "usd_per_1k_answers": 0.3158625,
   "usd_per_correct": 0.00034379591836734694,
   "avg_prompt_tokens": 930.025,
   "avg_completion_tokens": 293.93125,
   "avg_reasoning_tokens": 0,
   "token_basis": "measured",
   "latency_median_s": 2.635
  },
  "claude-haiku-5-5": {
   "label": "Claude Haiku 5.5",
   "n": 160,
   "correct": 148,
   "accuracy": 0.925,
   "accuracy_ci95": [
    0.8735,
    0.9566
   ],
   "usd_per_1k_answers": 0.484338125,
   "usd_per_correct": 0.0005236087837837838,
   "avg_prompt_tokens": 1402.81875,
   "avg_completion_tokens": 688.1125,
   "avg_reasoning_tokens": 0,
   "token_basis": "measured",
   "latency_median_s": 3.0149999999999997
  },
  "claude-fable-5-1": {
   "label": "Claude Fable 5.1",
   "n": 160,
   "correct": 154,
   "accuracy": 0.9625,
   "accuracy_ci95": [
    0.9206,
    0.9827
   ],
   "usd_per_1k_answers": 25.219437500000005,
   "usd_per_correct": 0.02620201298701299,
   "avg_prompt_tokens": 1402.81875,
   "avg_completion_tokens": 223.825,
   "avg_reasoning_tokens": 0,
   "token_basis": "measured",
   "latency_median_s": 2.7300000000000004
  },
  "claude:claude-fable-5-2": {
   "label": "Claude Fable 5.2",
   "n": 160,
   "correct": 156,
   "accuracy": 0.975,
   "accuracy_ci95": [
    0.9375,
    0.9902
   ],
   "usd_per_1k_answers": 10.8872,
   "usd_per_correct": 0.011166358974358973,
   "avg_prompt_tokens": 856.025,
   "avg_completion_tokens": 100.975,
   "avg_reasoning_tokens": 0,
   "token_basis": "estimated",
   "latency_median_s": null
  }
 },
 "table_all_three_tasks_proxy_models": {
  "claude-sonnet-5-5-high": {
   "label": "Claude Sonnet 5.5",
   "n": 240,
   "correct": 236,
   "accuracy": 0.9833333333333333,
   "accuracy_ci95": [
    0.9579,
    0.9935
   ],
   "usd_per_1k_answers": 5.331916666666667,
   "usd_per_correct": 0.005422288135593221,
   "avg_prompt_tokens": 1589.375,
   "avg_completion_tokens": 215.31666666666666,
   "avg_reasoning_tokens": 0,
   "token_basis": "measured",
   "latency_median_s": 2.57
  },
  "claude-opus-5-5-high": {
   "label": "Claude Opus 5.5",
   "n": 240,
   "correct": 235,
   "accuracy": 0.9791666666666666,
   "accuracy_ci95": [
    0.9522,
    0.9911
   ],
   "usd_per_1k_answers": 11.67275,
   "usd_per_correct": 0.011921106382978724,
   "avg_prompt_tokens": 1589.375,
   "avg_completion_tokens": 265.7625,
   "avg_reasoning_tokens": 0,
   "token_basis": "measured",
   "latency_median_s": 2.89
  },
  "gemini-3.8-flash-high": {
   "label": "Gemini 3.8 Flash",
   "n": 240,
   "correct": 236,
   "accuracy": 0.9833333333333333,
   "accuracy_ci95": [
    0.9579,
    0.9935
   ],
   "usd_per_1k_answers": 3.410875,
   "usd_per_correct": 0.0034686864406779662,
   "avg_prompt_tokens": 1088.7916666666667,
   "avg_completion_tokens": 691.8083333333333,
   "avg_reasoning_tokens": 595.575,
   "token_basis": "measured",
   "latency_median_s": 2.9699999999999998
  },
  "gemini-3.1-flash-lite": {
   "label": "Gemini 3.1 Flash-Lite",
   "n": 240,
   "correct": 221,
   "accuracy": 0.9208333333333333,
   "accuracy_ci95": [
    0.8797,
    0.9487
   ],
   "usd_per_1k_answers": 0.40424791666666665,
   "usd_per_correct": 0.0004390022624434389,
   "avg_prompt_tokens": 1088.7916666666667,
   "avg_completion_tokens": 88.03333333333333,
   "avg_reasoning_tokens": 0,
   "token_basis": "measured",
   "latency_median_s": 0.95
  },
  "gemini-4-argon-high": {
   "label": "Gemini 4 Argon",
   "n": 240,
   "correct": 234,
   "accuracy": 0.975,
   "accuracy_ci95": [
    0.9465,
    0.9885
   ],
   "usd_per_1k_answers": 3.808358333333333,
   "usd_per_correct": 0.003906008547008547,
   "avg_prompt_tokens": 1128.075,
   "avg_completion_tokens": 155.22083333333333,
   "avg_reasoning_tokens": 43.99583333333333,
   "token_basis": "measured",
   "latency_median_s": 7.045
  },
  "gpt-oss-120b-medium": {
   "label": "gpt-oss-120b",
   "n": 240,
   "correct": 219,
   "accuracy": 0.9125,
   "accuracy_ci95": [
    0.8699,
    0.9421
   ],
   "usd_per_1k_answers": 0.28465625000000006,
   "usd_per_correct": 0.0003119520547945206,
   "avg_prompt_tokens": 943.8583333333333,
   "avg_completion_tokens": 238.4625,
   "avg_reasoning_tokens": 0,
   "token_basis": "measured",
   "latency_median_s": 2.36
  },
  "claude-haiku-5-5": {
   "label": "Claude Haiku 5.5",
   "n": 240,
   "correct": 228,
   "accuracy": 0.95,
   "accuracy_ci95": [
    0.9147,
    0.9712
   ],
   "usd_per_1k_answers": 0.4449625,
   "usd_per_correct": 0.0004683815789473684,
   "avg_prompt_tokens": 1730.7083333333333,
   "avg_completion_tokens": 543.7833333333333,
   "avg_reasoning_tokens": 0,
   "token_basis": "measured",
   "latency_median_s": 2.3899999999999997
  },
  "claude-fable-5-1": {
   "label": "Claude Fable 5.1",
   "n": 240,
   "correct": 233,
   "accuracy": 0.9708333333333333,
   "accuracy_ci95": [
    0.941,
    0.9858
   ],
   "usd_per_1k_answers": 27.69291666666667,
   "usd_per_correct": 0.028524892703862662,
   "avg_prompt_tokens": 1730.7083333333333,
   "avg_completion_tokens": 207.71666666666667,
   "avg_reasoning_tokens": 0,
   "token_basis": "measured",
   "latency_median_s": 2.5199999999999996
  }
 },
 "per_task": {
  "support": {
   "claude-sonnet-5-5-high": {
    "label": "Claude Sonnet 5.5",
    "n": 80,
    "correct": 76,
    "accuracy": 0.95,
    "accuracy_ci95": [
     0.8784,
     0.9804
    ],
    "usd_per_1k_answers": 6.317100000000001,
    "usd_per_correct": 0.006649578947368421,
    "avg_prompt_tokens": 1850.2375,
    "avg_completion_tokens": 261.6625,
    "avg_reasoning_tokens": 0,
    "token_basis": "measured",
    "latency_median_s": 2.69
   },
   "claude-opus-5-5-high": {
    "label": "Claude Opus 5.5",
    "n": 80,
    "correct": 75,
    "accuracy": 0.9375,
    "accuracy_ci95": [
     0.8619,
     0.973
    ],
    "usd_per_1k_answers": 15.12995,
    "usd_per_correct": 0.016138613333333333,
    "avg_prompt_tokens": 1850.2375,
    "avg_completion_tokens": 386.45,
    "avg_reasoning_tokens": 0,
    "token_basis": "measured",
    "latency_median_s": 4.345
   },
   "gemini-3.8-flash-high": {
    "label": "Gemini 3.8 Flash",
    "n": 80,
    "correct": 76,
    "accuracy": 0.95,
    "accuracy_ci95": [
     0.8784,
     0.9804
    ],
    "usd_per_1k_answers": 4.6609125,
    "usd_per_correct": 0.004906223684210526,
    "avg_prompt_tokens": 1366.8,
    "avg_completion_tokens": 969.55,
    "avg_reasoning_tokens": 875.4625,
    "token_basis": "measured",
    "latency_median_s": 3.5300000000000002
   },
   "gemini-3.1-flash-lite": {
    "label": "Gemini 3.1 Flash-Lite",
    "n": 80,
    "correct": 64,
    "accuracy": 0.8,
    "accuracy_ci95": [
     0.6995,
     0.873
    ],
    "usd_per_1k_answers": 0.447525,
    "usd_per_correct": 0.00055940625,
    "avg_prompt_tokens": 1366.8,
    "avg_completion_tokens": 70.55,
    "avg_reasoning_tokens": 0,
    "token_basis": "measured",
    "latency_median_s": 0.98
   },
   "gemini-4-argon-high": {
    "label": "Gemini 4 Argon",
    "n": 80,
    "correct": 74,
    "accuracy": 0.925,
    "accuracy_ci95": [
     0.8459,
     0.9652
    ],
    "usd_per_1k_answers": 4.39675,
    "usd_per_correct": 0.004753243243243243,
    "avg_prompt_tokens": 1397.1875,
    "avg_completion_tokens": 160.2375,
    "avg_reasoning_tokens": 27.975,
    "token_basis": "measured",
    "latency_median_s": 7.14
   },
   "gpt-oss-120b-medium": {
    "label": "gpt-oss-120b",
    "n": 80,
    "correct": 67,
    "accuracy": 0.8375,
    "accuracy_ci95": [
     0.7416,
     0.9025
    ],
    "usd_per_1k_answers": 0.33637875,
    "usd_per_correct": 0.00040164626865671636,
    "avg_prompt_tokens": 1328.125,
    "avg_completion_tokens": 228.6,
    "avg_reasoning_tokens": 0,
    "token_basis": "measured",
    "latency_median_s": 2.4450000000000003
   },
   "claude-haiku-5-5": {
    "label": "Claude Haiku 5.5",
    "n": 80,
    "correct": 68,
    "accuracy": 0.85,
    "accuracy_ci95": [
     0.7559,
     0.9121
    ],
    "usd_per_1k_answers": 0.607205,
    "usd_per_correct": 0.0007143588235294118,
    "avg_prompt_tokens": 1975.2375,
    "avg_completion_tokens": 819.3625,
    "avg_reasoning_tokens": 0,
    "token_basis": "measured",
    "latency_median_s": 3.9000000000000004
   },
   "claude-fable-5-1": {
    "label": "Claude Fable 5.1",
    "n": 80,
    "correct": 74,
    "accuracy": 0.925,
    "accuracy_ci95": [
     0.8459,
     0.9652
    ],
    "usd_per_1k_answers": 33.62425,
    "usd_per_correct": 0.036350540540540544,
    "avg_prompt_tokens": 1975.2375,
    "avg_completion_tokens": 277.4375,
    "avg_reasoning_tokens": 0,
    "token_basis": "measured",
    "latency_median_s": 3.355
   },
   "claude:claude-fable-5-2": {
    "label": "Claude Fable 5.2",
    "n": 80,
    "correct": 76,
    "accuracy": 0.95,
    "accuracy_ci95": [
     0.8784,
     0.9804
    ],
    "usd_per_1k_answers": 14.8175,
    "usd_per_correct": 0.015597368421052632,
    "avg_prompt_tokens": 1254.125,
    "avg_completion_tokens": 119.6125,
    "avg_reasoning_tokens": 0,
    "token_basis": "estimated",
    "latency_median_s": null
   }
  },
  "tools": {
   "claude-sonnet-5-5-high": {
    "label": "Claude Sonnet 5.5",
    "n": 80,
    "correct": 80,
    "accuracy": 1.0,
    "accuracy_ci95": [
     0.9542,
     1.0
    ],
    "usd_per_1k_answers": 5.7561,
    "usd_per_correct": 0.0057561,
    "avg_prompt_tokens": 2212.4875,
    "avg_completion_tokens": 133.1125,
    "avg_reasoning_tokens": 0,
    "token_basis": "measured",
    "latency_median_s": 1.8849999999999998
   },
   "claude-opus-5-5-high": {
    "label": "Claude Opus 5.5",
    "n": 80,
    "correct": 80,
    "accuracy": 1.0,
    "accuracy_ci95": [
     0.9542,
     1.0
    ],
    "usd_per_1k_answers": 11.8387,
    "usd_per_correct": 0.0118387,
    "avg_prompt_tokens": 2212.4875,
    "avg_completion_tokens": 149.4375,
    "avg_reasoning_tokens": 0,
    "token_basis": "measured",
    "latency_median_s": 2.24
   },
   "gemini-3.8-flash-high": {
    "label": "Gemini 3.8 Flash",
    "n": 80,
    "correct": 80,
    "accuracy": 1.0,
    "accuracy_ci95": [
     0.9542,
     1.0
    ],
    "usd_per_1k_answers": 2.98528125,
    "usd_per_correct": 0.00298528125,
    "avg_prompt_tokens": 1368.9375,
    "avg_completion_tokens": 522.2875,
    "avg_reasoning_tokens": 479.975,
    "token_basis": "measured",
    "latency_median_s": 2.585
   },
   "gemini-3.1-flash-lite": {
    "label": "Gemini 3.1 Flash-Lite",
    "n": 80,
    "correct": 77,
    "accuracy": 0.9625,
    "accuracy_ci95": [
     0.8955,
     0.9872
    ],
    "usd_per_1k_answers": 0.40332187500000005,
    "usd_per_correct": 0.00041903571428571435,
    "avg_prompt_tokens": 1368.9375,
    "avg_completion_tokens": 40.725,
    "avg_reasoning_tokens": 0,
    "token_basis": "measured",
    "latency_median_s": 0.84
   },
   "gemini-4-argon-high": {
    "label": "Gemini 4 Argon",
    "n": 80,
    "correct": 80,
    "accuracy": 1.0,
    "accuracy_ci95": [
     0.9542,
     1.0
    ],
    "usd_per_1k_answers": 3.5787750000000003,
    "usd_per_correct": 0.003578775,
    "avg_prompt_tokens": 1427.575,
    "avg_completion_tokens": 72.3625,
    "avg_reasoning_tokens": 21.4375,
    "token_basis": "measured",
    "latency_median_s": 6.390000000000001
   },
   "gpt-oss-120b-medium": {
    "label": "gpt-oss-120b",
    "n": 80,
    "correct": 72,
    "accuracy": 0.9,
    "accuracy_ci95": [
     0.8149,
     0.9485
    ],
    "usd_per_1k_answers": 0.22224375,
    "usd_per_correct": 0.0002469375,
    "avg_prompt_tokens": 971.525,
    "avg_completion_tokens": 127.525,
    "avg_reasoning_tokens": 0,
    "token_basis": "measured",
    "latency_median_s": 1.69
   },
   "claude-haiku-5-5": {
    "label": "Claude Haiku 5.5",
    "n": 80,
    "correct": 80,
    "accuracy": 1.0,
    "accuracy_ci95": [
     0.9542,
     1.0
    ],
    "usd_per_1k_answers": 0.36621125,
    "usd_per_correct": 0.00036621125,
    "avg_prompt_tokens": 2386.4875,
    "avg_completion_tokens": 255.125,
    "avg_reasoning_tokens": 0,
    "token_basis": "measured",
    "latency_median_s": 1.545
   },
   "claude-fable-5-1": {
    "label": "Claude Fable 5.1",
    "n": 80,
    "correct": 79,
    "accuracy": 0.9875,
    "accuracy_ci95": [
     0.9325,
     0.9978
    ],
    "usd_per_1k_answers": 32.639875,
    "usd_per_correct": 0.03305303797468354,
    "avg_prompt_tokens": 2386.4875,
    "avg_completion_tokens": 175.5,
    "avg_reasoning_tokens": 0,
    "token_basis": "measured",
    "latency_median_s": 2.325
   }
  },
  "extract": {
   "claude-sonnet-5-5-high": {
    "label": "Claude Sonnet 5.5",
    "n": 80,
    "correct": 80,
    "accuracy": 1.0,
    "accuracy_ci95": [
     0.9542,
     1.0
    ],
    "usd_per_1k_answers": 3.9225499999999998,
    "usd_per_correct": 0.0039225499999999995,
    "avg_prompt_tokens": 705.4,
    "avg_completion_tokens": 251.175,
    "avg_reasoning_tokens": 0,
    "token_basis": "measured",
    "latency_median_s": 2.885
   },
   "claude-opus-5-5-high": {
    "label": "Claude Opus 5.5",
    "n": 80,
    "correct": 80,
    "accuracy": 1.0,
    "accuracy_ci95": [
     0.9542,
     1.0
    ],
    "usd_per_1k_answers": 8.0496,
    "usd_per_correct": 0.0080496,
    "avg_prompt_tokens": 705.4,
    "avg_completion_tokens": 261.4,
    "avg_reasoning_tokens": 0,
    "token_basis": "measured",
    "latency_median_s": 2.84
   },
   "gemini-3.8-flash-high": {
    "label": "Gemini 3.8 Flash",
    "n": 80,
    "correct": 80,
    "accuracy": 1.0,
    "accuracy_ci95": [
     0.9542,
     1.0
    ],
    "usd_per_1k_answers": 2.58643125,
    "usd_per_correct": 0.00258643125,
    "avg_prompt_tokens": 530.6375,
    "avg_completion_tokens": 583.5875,
    "avg_reasoning_tokens": 431.2875,
    "token_basis": "measured",
    "latency_median_s": 2.87
   },
   "gemini-3.1-flash-lite": {
    "label": "Gemini 3.1 Flash-Lite",
    "n": 80,
    "correct": 80,
    "accuracy": 1.0,
    "accuracy_ci95": [
     0.9542,
     1.0
    ],
    "usd_per_1k_answers": 0.36189687499999995,
    "usd_per_correct": 0.000361896875,
    "avg_prompt_tokens": 530.6375,
    "avg_completion_tokens": 152.825,
    "avg_reasoning_tokens": 0,
    "token_basis": "measured",
    "latency_median_s": 0.995
   },
   "gemini-4-argon-high": {
    "label": "Gemini 4 Argon",
    "n": 80,
    "correct": 80,
    "accuracy": 1.0,
    "accuracy_ci95": [
     0.9542,
     1.0
    ],
    "usd_per_1k_answers": 3.44955,
    "usd_per_correct": 0.00344955,
    "avg_prompt_tokens": 559.4625,
    "avg_completion_tokens": 233.0625,
    "avg_reasoning_tokens": 82.575,
    "token_basis": "measured",
    "latency_median_s": 7.45
   },
   "gpt-oss-120b-medium": {
    "label": "gpt-oss-120b",
    "n": 80,
    "correct": 80,
    "accuracy": 1.0,
    "accuracy_ci95": [
     0.9542,
     1.0
    ],
    "usd_per_1k_answers": 0.29534625,
    "usd_per_correct": 0.00029534624999999996,
    "avg_prompt_tokens": 531.925,
    "avg_completion_tokens": 359.2625,
    "avg_reasoning_tokens": 0,
    "token_basis": "measured",
    "latency_median_s": 2.76
   },
   "claude-haiku-5-5": {
    "label": "Claude Haiku 5.5",
    "n": 80,
    "correct": 80,
    "accuracy": 1.0,
    "accuracy_ci95": [
     0.9542,
     1.0
    ],
    "usd_per_1k_answers": 0.36147125,
    "usd_per_correct": 0.00036147125,
    "avg_prompt_tokens": 830.4,
    "avg_completion_tokens": 556.8625,
    "avg_reasoning_tokens": 0,
    "token_basis": "measured",
    "latency_median_s": 2.4050000000000002
   },
   "claude-fable-5-1": {
    "label": "Claude Fable 5.1",
    "n": 80,
    "correct": 80,
    "accuracy": 1.0,
    "accuracy_ci95": [
     0.9542,
     1.0
    ],
    "usd_per_1k_answers": 16.814625,
    "usd_per_correct": 0.016814625,
    "avg_prompt_tokens": 830.4,
    "avg_completion_tokens": 170.2125,
    "avg_reasoning_tokens": 0,
    "token_basis": "measured",
    "latency_median_s": 1.68
   },
   "claude:claude-fable-5-2": {
    "label": "Claude Fable 5.2",
    "n": 80,
    "correct": 80,
    "accuracy": 1.0,
    "accuracy_ci95": [
     0.9542,
     1.0
    ],
    "usd_per_1k_answers": 6.9569,
    "usd_per_correct": 0.0069569,
    "avg_prompt_tokens": 457.925,
    "avg_completion_tokens": 82.3375,
    "avg_reasoning_tokens": 0,
    "token_basis": "estimated",
    "latency_median_s": null
   }
  }
 },
 "routing_proxy_models_3_tasks": {
  "models": [
   "claude-sonnet-5-5-high",
   "claude-opus-5-5-high",
   "gemini-3.8-flash-high",
   "gemini-3.1-flash-lite",
   "gemini-4-argon-high",
   "gpt-oss-120b-medium",
   "claude-haiku-5-5",
   "claude-fable-5-1"
  ],
  "tasks": [
   "support",
   "tools",
   "extract"
  ],
  "n_items": 240,
  "method": "50/50 stratified-by-task split of the items, repeated 500 times; calibration: per segment (support intent / tools kind / extract layout, >=3 calibration items) pick the cheapest model whose calibration accuracy equals the best any model reaches in that segment, else the best overall model; held-out: route by segment and compare with always using the best overall model (chosen on calibration).",
  "segment": {
   "repeats": 500,
   "savings_mean": 0.8269909767668255,
   "savings_p05": 0.6721565463294665,
   "savings_p95": 0.9407902337582317,
   "quality_delta_pp_mean": -1.3783333333333334,
   "quality_delta_pp_p05": -4.166666666666666,
   "quality_delta_pp_p95": 0.8333333333333334,
   "best_model_counts": {
    "Claude Sonnet 5.5": 156,
    "Gemini 3.8 Flash": 344
   },
   "route_acc_mean": 0.9661166666666666,
   "best_acc_mean": 0.9799,
   "route_usd_per_1k_mean": 0.6450595483333333,
   "best_usd_per_1k_mean": 3.9890065666666668
  },
  "task": {
   "repeats": 500,
   "savings_mean": 0.5115347906230914,
   "savings_p05": 0.4006938082541418,
   "savings_p95": 0.5737938485404932,
   "quality_delta_pp_mean": -0.4,
   "quality_delta_pp_p05": -2.5,
   "quality_delta_pp_p95": 0.0,
   "best_model_counts": {
    "Claude Sonnet 5.5": 156,
    "Gemini 3.8 Flash": 344
   },
   "route_acc_mean": 0.9759,
   "best_acc_mean": 0.9799,
   "route_usd_per_1k_mean": 1.9176261791666664,
   "best_usd_per_1k_mean": 3.9890065666666668
  },
  "oracle_upper_bound_not_achievable": {
   "savings": 0.897997319600519,
   "accuracy": 0.9958333333333333,
   "best_overall": "Claude Sonnet 5.5"
  }
 },
 "routing_all_models_paired_subset": {
  "models": [
   "claude-sonnet-5-5-high",
   "claude-opus-5-5-high",
   "gemini-3.8-flash-high",
   "gemini-3.1-flash-lite",
   "gemini-4-argon-high",
   "gpt-oss-120b-medium",
   "claude-haiku-5-5",
   "claude-fable-5-1",
   "claude:claude-fable-5-2"
  ],
  "tasks": [
   "support",
   "extract"
  ],
  "n_items": 160,
  "method": "50/50 stratified-by-task split of the items, repeated 500 times; calibration: per segment (support intent / tools kind / extract layout, >=3 calibration items) pick the cheapest model whose calibration accuracy equals the best any model reaches in that segment, else the best overall model; held-out: route by segment and compare with always using the best overall model (chosen on calibration).",
  "segment": {
   "repeats": 500,
   "savings_mean": 0.7825448078082639,
   "savings_p05": 0.5813925377943415,
   "savings_p95": 0.9322944385481878,
   "quality_delta_pp_mean": -1.8599999999999999,
   "quality_delta_pp_p05": -6.25,
   "quality_delta_pp_p95": 1.25,
   "best_model_counts": {
    "Claude Sonnet 5.5": 156,
    "Claude Fable 5.2": 34,
    "Gemini 3.8 Flash": 304,
    "gpt-oss-120b": 6
   },
   "route_acc_mean": 0.94905,
   "best_acc_mean": 0.96765,
   "route_usd_per_1k_mean": 0.8882200975000001,
   "best_usd_per_1k_mean": 4.52624310875
  },
  "task": {
   "repeats": 500,
   "savings_mean": 0.3266518140845203,
   "savings_p05": 0.2707298148939876,
   "savings_p95": 0.3711140378290697,
   "quality_delta_pp_mean": 0.0,
   "quality_delta_pp_p05": 0.0,
   "quality_delta_pp_p95": 0.0,
   "best_model_counts": {
    "Claude Sonnet 5.5": 156,
    "Claude Fable 5.2": 34,
    "Gemini 3.8 Flash": 304,
    "gpt-oss-120b": 6
   },
   "route_acc_mean": 0.96765,
   "best_acc_mean": 0.96765,
   "route_usd_per_1k_mean": 3.0395126637500005,
   "best_usd_per_1k_mean": 4.52624310875
  },
  "oracle_upper_bound_not_achievable": {
   "savings": 0.903007610123393,
   "accuracy": 0.99375,
   "best_overall": "Claude Sonnet 5.5"
  }
 },
 "caching": {
  "measured_probe": {
   "Claude Sonnet 5.5|plain": {
    "repeat_calls": 5,
    "repeat_calls_with_cache_hit": 5,
    "cached_tokens_per_repeat": [
     6721,
     6721,
     6721,
     6721,
     6721
    ],
    "prompt_tokens_per_repeat": [
     6817,
     6792,
     6803,
     6826,
     7071
    ],
    "input_cost_without_cache_usd": 0.068618,
    "input_cost_with_reported_cache_usd": 0.0047685,
    "input_saving_pct": 0.9305065726194293,
    "label": "MEASURED: cached_tokens reported in usage x list prices"
   },
   "Claude Sonnet 5.5|cache_control": {
    "repeat_calls": 5,
    "repeat_calls_with_cache_hit": 5,
    "cached_tokens_per_repeat": [
     6720,
     6720,
     6720,
     6720,
     6720
    ],
    "prompt_tokens_per_repeat": [
     6816,
     6791,
     6802,
     6825,
     7070
    ],
    "input_cost_without_cache_usd": 0.068608,
    "input_cost_with_reported_cache_usd": 0.004768,
    "input_saving_pct": 0.9305037313432836,
    "label": "MEASURED: cached_tokens reported in usage x list prices"
   },
   "Claude Opus 5.5|plain": {
    "repeat_calls": 5,
    "repeat_calls_with_cache_hit": 5,
    "cached_tokens_per_repeat": [
     6724,
     6724,
     6724,
     6724,
     6724
    ],
    "prompt_tokens_per_repeat": [
     6820,
     6795,
     6806,
     6829,
     7074
    ],
    "input_cost_without_cache_usd": 0.137296,
    "input_cost_with_reported_cache_usd": 0.00954,
    "input_saving_pct": 0.9305150914811794,
    "label": "MEASURED: cached_tokens reported in usage x list prices"
   },
   "Claude Opus 5.5|cache_control": {
    "repeat_calls": 5,
    "repeat_calls_with_cache_hit": 5,
    "cached_tokens_per_repeat": [
     6720,
     6720,
     6720,
     6720,
     6720
    ],
    "prompt_tokens_per_repeat": [
     6816,
     6791,
     6802,
     6825,
     7070
    ],
    "input_cost_without_cache_usd": 0.137216,
    "input_cost_with_reported_cache_usd": 0.009536,
    "input_saving_pct": 0.9305037313432836,
    "label": "MEASURED: cached_tokens reported in usage x list prices"
   },
   "Gemini 3.8 Flash|plain": {
    "repeat_calls": 5,
    "repeat_calls_with_cache_hit": 0,
    "cached_tokens_per_repeat": [
     0,
     0,
     0,
     0,
     0
    ],
    "prompt_tokens_per_repeat": [
     5729,
     5708,
     5712,
     5724,
     5972
    ],
    "input_cost_without_cache_usd": 0.02163375,
    "input_cost_with_reported_cache_usd": 0.02163375,
    "input_saving_pct": 0.0,
    "label": "no cached tokens reported by the proxy (saving not measured)"
   },
   "Gemini 3.1 Flash-Lite|plain": {
    "repeat_calls": 5,
    "repeat_calls_with_cache_hit": 0,
    "cached_tokens_per_repeat": [
     0,
     0,
     0,
     0,
     0
    ],
    "prompt_tokens_per_repeat": [
     5727,
     5706,
     5710,
     5722,
     5970
    ],
    "input_cost_without_cache_usd": 0.00720875,
    "input_cost_with_reported_cache_usd": 0.00720875,
    "input_saving_pct": 0.0,
    "label": "no cached tokens reported by the proxy (saving not measured)"
   },
   "Gemini 4 Argon|plain": {
    "repeat_calls": 5,
    "repeat_calls_with_cache_hit": 0,
    "cached_tokens_per_repeat": [
     0,
     0,
     0,
     0,
     0
    ],
    "prompt_tokens_per_repeat": [
     5733,
     5712,
     5716,
     5728,
     5976
    ],
    "input_cost_without_cache_usd": 0.05773,
    "input_cost_with_reported_cache_usd": 0.05773,
    "input_saving_pct": 0.0,
    "label": "no cached tokens reported by the proxy (saving not measured)"
   },
   "gpt-oss-120b|plain": {
    "repeat_calls": 5,
    "repeat_calls_with_cache_hit": 4,
    "cached_tokens_per_repeat": [
     4928,
     64,
     4928,
     4928,
     4928
    ],
    "prompt_tokens_per_repeat": [
     5005,
     4983,
     4988,
     5000,
     5195
    ],
    "input_cost_without_cache_usd": 0.0037756499999999998,
    "input_cost_with_reported_cache_usd": 0.0022924499999999997,
    "input_saving_pct": 0.39283302212864013,
    "label": "MEASURED: cached_tokens reported in usage x list prices"
   }
  },
  "main_run_reported_cached_tokens": {
   "Claude Sonnet 5.5": {
    "prompt_tokens": 381450,
    "cached_tokens_reported": 248946,
    "list_cost_without_cache_usd": 1.27966,
    "measured_saving_usd": 0.47299739999999996,
    "measured_saving_pct": 0.36962740102839814,
    "label": "MEASURED cached tokens (usage.prompt_tokens_details.cached_tokens) x list price difference; gross of any cache-write premium, which the proxy does not report"
   },
   "Claude Opus 5.5": {
    "prompt_tokens": 381450,
    "cached_tokens_reported": 248420,
    "list_cost_without_cache_usd": 2.80146,
    "measured_saving_usd": 0.943996,
    "measured_saving_pct": 0.3369657250148137,
    "label": "MEASURED cached tokens (usage.prompt_tokens_details.cached_tokens) x list price difference; gross of any cache-write premium, which the proxy does not report"
   },
   "Gemini 3.8 Flash": {
    "prompt_tokens": 261310,
    "cached_tokens_reported": 0,
    "list_cost_without_cache_usd": 0.8186100000000001,
    "measured_saving_usd": 0.0,
    "measured_saving_pct": 0.0,
    "label": "no cached tokens reported (measured saving = 0)"
   },
   "Gemini 3.1 Flash-Lite": {
    "prompt_tokens": 261310,
    "cached_tokens_reported": 0,
    "list_cost_without_cache_usd": 0.0970195,
    "measured_saving_usd": 0.0,
    "measured_saving_pct": 0.0,
    "label": "no cached tokens reported (measured saving = 0)"
   },
   "Gemini 4 Argon": {
    "prompt_tokens": 270738,
    "cached_tokens_reported": 0,
    "list_cost_without_cache_usd": 0.914006,
    "measured_saving_usd": 0.0,
    "measured_saving_pct": 0.0,
    "label": "no cached tokens reported (measured saving = 0)"
   },
   "gpt-oss-120b": {
    "prompt_tokens": 226526,
    "cached_tokens_reported": 141376,
    "list_cost_without_cache_usd": 0.0683175,
    "measured_saving_usd": 0.010603199999999998,
    "measured_saving_pct": 0.155204742562301,
    "label": "MEASURED cached tokens (usage.prompt_tokens_details.cached_tokens) x list price difference; gross of any cache-write premium, which the proxy does not report"
   },
   "Claude Haiku 5.5": {
    "prompt_tokens": 415370,
    "cached_tokens_reported": 126164,
    "list_cost_without_cache_usd": 0.106791,
    "measured_saving_usd": 0.011354760000000002,
    "measured_saving_pct": 0.10632693766328626,
    "label": "MEASURED cached tokens (usage.prompt_tokens_details.cached_tokens) x list price difference; gross of any cache-write premium, which the proxy does not report"
   },
   "Claude Fable 5.1": {
    "prompt_tokens": 415370,
    "cached_tokens_reported": 126164,
    "list_cost_without_cache_usd": 6.6463,
    "measured_saving_usd": 1.230099,
    "measured_saving_pct": 0.18508027022553902,
    "label": "MEASURED cached tokens (usage.prompt_tokens_details.cached_tokens) x list price difference; gross of any cache-write premium, which the proxy does not report"
   }
  },
  "computed_workload": {
   "Claude Sonnet 5.5": {
    "support": {
     "approx_shared_prefix_tokens": 1575,
     "n_items": 80,
     "total_cost_usd": 0.505368,
     "computed_saving_usd": 0.23560583380491013,
     "computed_saving_pct_of_total_cost": 0.4662064748953438,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    },
    "tools": {
     "approx_shared_prefix_tokens": 1382,
     "n_items": 80,
     "total_cost_usd": 0.460488,
     "computed_saving_usd": 0.20676157395613493,
     "computed_saving_pct_of_total_cost": 0.449005346406714,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    },
    "extract": {
     "approx_shared_prefix_tokens": 377,
     "n_items": 80,
     "total_cost_usd": 0.31380399999999997,
     "computed_saving_usd": 0.0563400253231028,
     "computed_saving_pct_of_total_cost": 0.17953890110738807,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    }
   },
   "Claude Opus 5.5": {
    "support": {
     "approx_shared_prefix_tokens": 1577,
     "n_items": 80,
     "total_cost_usd": 1.210396,
     "computed_saving_usd": 0.47197281924753626,
     "computed_saving_pct_of_total_cost": 0.3899325669016886,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    },
    "tools": {
     "approx_shared_prefix_tokens": 1377,
     "n_items": 80,
     "total_cost_usd": 0.947096,
     "computed_saving_usd": 0.4120249684445967,
     "computed_saving_pct_of_total_cost": 0.43504034273674125,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    },
    "extract": {
     "approx_shared_prefix_tokens": 377,
     "n_items": 80,
     "total_cost_usd": 0.643968,
     "computed_saving_usd": 0.1126800506462056,
     "computed_saving_pct_of_total_cost": 0.17497771728751368,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    }
   },
   "Gemini 3.8 Flash": {
    "support": {
     "approx_shared_prefix_tokens": 1153,
     "n_items": 80,
     "total_cost_usd": 0.372873,
     "computed_saving_usd": 0.06150776866306759,
     "computed_saving_pct_of_total_cost": 0.16495634884549856,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    },
    "tools": {
     "approx_shared_prefix_tokens": 854,
     "n_items": 80,
     "total_cost_usd": 0.2388225,
     "computed_saving_usd": 0.04555008196652059,
     "computed_saving_pct_of_total_cost": 0.1907277662972316,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    },
    "extract": {
     "approx_shared_prefix_tokens": 282,
     "n_items": 80,
     "total_cost_usd": 0.2069145,
     "computed_saving_usd": 0.015063998692185217,
     "computed_saving_pct_of_total_cost": 0.07280301135099385,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    }
   },
   "Gemini 3.1 Flash-Lite": {
    "support": {
     "approx_shared_prefix_tokens": 1152,
     "n_items": 80,
     "total_cost_usd": 0.035802,
     "computed_saving_usd": 0.020478982551262976,
     "computed_saving_pct_of_total_cost": 0.5720066630708612,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    },
    "tools": {
     "approx_shared_prefix_tokens": 851,
     "n_items": 80,
     "total_cost_usd": 0.03226575,
     "computed_saving_usd": 0.015121495697689,
     "computed_saving_pct_of_total_cost": 0.46865470964378636,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    },
    "extract": {
     "approx_shared_prefix_tokens": 282,
     "n_items": 80,
     "total_cost_usd": 0.02895175,
     "computed_saving_usd": 0.005014352749704566,
     "computed_saving_pct_of_total_cost": 0.17319687928033942,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    }
   },
   "Gemini 4 Argon": {
    "support": {
     "approx_shared_prefix_tokens": 1179,
     "n_items": 80,
     "total_cost_usd": 0.35174,
     "computed_saving_usd": 0.17698888267929413,
     "computed_saving_pct_of_total_cost": 0.5031809935727928,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    },
    "tools": {
     "approx_shared_prefix_tokens": 888,
     "n_items": 80,
     "total_cost_usd": 0.286302,
     "computed_saving_usd": 0.1332539566176513,
     "computed_saving_pct_of_total_cost": 0.46543145565749205,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    },
    "extract": {
     "approx_shared_prefix_tokens": 298,
     "n_items": 80,
     "total_cost_usd": 0.275964,
     "computed_saving_usd": 0.04467165184046271,
     "computed_saving_pct_of_total_cost": 0.1618749251368393,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    }
   },
   "gpt-oss-120b": {
    "support": {
     "approx_shared_prefix_tokens": 1129,
     "n_items": 80,
     "total_cost_usd": 0.026910299999999998,
     "computed_saving_usd": 0.006686436866057761,
     "computed_saving_pct_of_total_cost": 0.24847128668419755,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    },
    "tools": {
     "approx_shared_prefix_tokens": 609,
     "n_items": 80,
     "total_cost_usd": 0.0177795,
     "computed_saving_usd": 0.0036088603435239755,
     "computed_saving_pct_of_total_cost": 0.20297873075868136,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    },
    "extract": {
     "approx_shared_prefix_tokens": 284,
     "n_items": 80,
     "total_cost_usd": 0.023627699999999998,
     "computed_saving_usd": 0.0016823586942654718,
     "computed_saving_pct_of_total_cost": 0.07120281255752663,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    }
   },
   "Claude Haiku 5.5": {
    "support": {
     "approx_shared_prefix_tokens": 1688,
     "n_items": 80,
     "total_cost_usd": 0.0485764,
     "computed_saving_usd": 0.011960276038650818,
     "computed_saving_pct_of_total_cost": 0.24621577635746614,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    },
    "tools": {
     "approx_shared_prefix_tokens": 1484,
     "n_items": 80,
     "total_cost_usd": 0.0292969,
     "computed_saving_usd": 0.010517090419843942,
     "computed_saving_pct_of_total_cost": 0.3589830466651401,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    },
    "extract": {
     "approx_shared_prefix_tokens": 444,
     "n_items": 80,
     "total_cost_usd": 0.0289177,
     "computed_saving_usd": 0.0031447343648196555,
     "computed_saving_pct_of_total_cost": 0.10874773459921278,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    }
   },
   "Claude Fable 5.1": {
    "support": {
     "approx_shared_prefix_tokens": 1688,
     "n_items": 80,
     "total_cost_usd": 2.68994,
     "computed_saving_usd": 1.296048260927899,
     "computed_saving_pct_of_total_cost": 0.4818130742425106,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    },
    "tools": {
     "approx_shared_prefix_tokens": 1484,
     "n_items": 80,
     "total_cost_usd": 2.61119,
     "computed_saving_usd": 1.1396607155730678,
     "computed_saving_pct_of_total_cost": 0.43645261952330844,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    },
    "extract": {
     "approx_shared_prefix_tokens": 444,
     "n_items": 80,
     "total_cost_usd": 1.34517,
     "computed_saving_usd": 0.34077202661824846,
     "computed_saving_pct_of_total_cost": 0.253330082159317,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    }
   },
   "GPT-6.1 Sol": {},
   "GPT-6.1 Terra": {},
   "GPT-6 Luna": {},
   "GPT-5.6 Sol": {},
   "Claude Fable 5.2": {
    "support": {
     "approx_shared_prefix_tokens": 1063,
     "n_items": 80,
     "total_cost_usd": 1.1854,
     "computed_saving_usd": 0.6528946,
     "computed_saving_pct_of_total_cost": 0.5507799898768349,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    },
    "extract": {
     "approx_shared_prefix_tokens": 244,
     "n_items": 80,
     "total_cost_usd": 0.556552,
     "computed_saving_usd": 0.1498648,
     "computed_saving_pct_of_total_cost": 0.26927367074415326,
     "label": "COMPUTED from list prices, not measured (upper bound: every repeat hits the cache; prefix may be under a provider's minimum cacheable length)"
    }
   }
  }
 },
 "sensitivity_unicode_dash_space_normalised_support_extract": {
  "Claude Sonnet 5.5": {
   "correct_as_scored": 156,
   "correct_after_normalising": 156,
   "n": 160
  },
  "Claude Opus 5.5": {
   "correct_as_scored": 155,
   "correct_after_normalising": 155,
   "n": 160
  },
  "Gemini 3.8 Flash": {
   "correct_as_scored": 156,
   "correct_after_normalising": 156,
   "n": 160
  },
  "Gemini 3.1 Flash-Lite": {
   "correct_as_scored": 144,
   "correct_after_normalising": 144,
   "n": 160
  },
  "Gemini 4 Argon": {
   "correct_as_scored": 154,
   "correct_after_normalising": 154,
   "n": 160
  },
  "gpt-oss-120b": {
   "correct_as_scored": 147,
   "correct_after_normalising": 150,
   "n": 160
  },
  "Claude Haiku 5.5": {
   "correct_as_scored": 148,
   "correct_after_normalising": 148,
   "n": 160
  },
  "Claude Fable 5.1": {
   "correct_as_scored": 154,
   "correct_after_normalising": 154,
   "n": 160
  },
  "Claude Fable 5.2": {
   "correct_as_scored": 156,
   "correct_after_normalising": 156,
   "n": 160
  }
 },
 "judge_agreement_opus_vs_carbon_on_migrationbench_replies": {
  "n": 480,
  "agree": 456,
  "note": "primary judge for CostBench = gemini-4-carbon-high (Opus hit proxy quota HTTP 429 mid-run); Opus labels exist only for the original MigrationBench replies"
 },
 "second_judge_claude_opus_5_5": {
  "n_common": 240,
  "agree": 231,
  "per_model": {
   "Claude Haiku 5.5": {
    "n": 80,
    "primary_pass": 67,
    "second_pass": 65
   },
   "Claude Fable 5.1": {
    "n": 80,
    "primary_pass": 74,
    "second_pass": 67
   },
   "Claude Fable 5.2": {
    "n": 80,
    "primary_pass": 76,
    "second_pass": 76
   }
  }
 }
}