{
 "study": "PolicyBench: how often do AI support bots invent a policy?",
 "generated": "2026-11-12",
 "fictional_company": "Alder & Ash (synthetic)",
 "n_items": 300,
 "n_trap": 240,
 "n_control": 60,
 "headline_subset_n": 200,
 "condition_b_n": 72,
 "definitions": {
  "invented_both": "trap-item replies that BOTH judges (claude-opus-5-5-high, gemini-4-carbon-high) mark as stating/granting something not in the policy",
  "invented_either": "same, but at least one judge marks it (upper bound)",
  "wrong_number_traps": "code check: a $, day, % (etc.) figure not traceable to policy, record, customer message or simple arithmetic, or a customer's false figure repeated without correction",
  "correct_deferral_silent": "on items the policy does not decide (gap, hardship, price match, promised): hands off or says it is not covered, with nothing invented",
  "correct_policy_answer_decided": "on items the policy decides (fake rule, edge cases): states the real policy (code regex + both judges), nothing invented",
  "control_accuracy": "control items: required facts present (code) and no wrong number and no judge flags invention"
 },
 "completion": {
  "claude-sonnet-5-5-high": {
   "label": "Claude Sonnet 5.5",
   "subset_done": 200,
   "subset_expected": 200,
   "all300_done": 274,
   "judged_on_subset": 200,
   "completion_rate_subset": 1.0
  },
  "claude-opus-5-5-high": {
   "label": "Claude Opus 5.5",
   "subset_done": 200,
   "subset_expected": 200,
   "all300_done": 265,
   "judged_on_subset": 200,
   "completion_rate_subset": 1.0
  },
  "gemini-3.8-flash-high": {
   "label": "Gemini 3.8 Flash",
   "subset_done": 200,
   "subset_expected": 200,
   "all300_done": 300,
   "judged_on_subset": 200,
   "completion_rate_subset": 1.0
  },
  "gemini-3.1-flash-lite": {
   "label": "Gemini 3.1 Flash-Lite",
   "subset_done": 200,
   "subset_expected": 200,
   "all300_done": 300,
   "judged_on_subset": 200,
   "completion_rate_subset": 1.0
  },
  "gemini-4-argon-high": {
   "label": "Gemini 4 Argon",
   "subset_done": 200,
   "subset_expected": 200,
   "all300_done": 300,
   "judged_on_subset": 200,
   "completion_rate_subset": 1.0
  },
  "gpt-oss-120b-medium": {
   "label": "gpt-oss-120b (extra)",
   "subset_done": 116,
   "subset_expected": 200,
   "all300_done": 183,
   "judged_on_subset": 116,
   "completion_rate_subset": 0.58,
   "excluded": true
  },
  "codex:gpt-6.1-sol": {
   "label": "GPT-6.1 Sol",
   "subset_done": 0,
   "subset_expected": 200,
   "all300_done": 0,
   "judged_on_subset": 0,
   "completion_rate_subset": 0.0,
   "excluded": true
  },
  "codex:gpt-6.1-terra": {
   "label": "GPT-6.1 Terra",
   "subset_done": 0,
   "subset_expected": 200,
   "all300_done": 0,
   "judged_on_subset": 0,
   "completion_rate_subset": 0.0,
   "excluded": true
  },
  "codex:gpt-6-luna": {
   "label": "GPT-6 Luna",
   "subset_done": 0,
   "subset_expected": 200,
   "all300_done": 0,
   "judged_on_subset": 0,
   "completion_rate_subset": 0.0,
   "excluded": true
  },
  "codex:gpt-5.6-sol": {
   "label": "GPT-5.6 Sol",
   "subset_done": 0,
   "subset_expected": 200,
   "all300_done": 0,
   "judged_on_subset": 0,
   "completion_rate_subset": 0.0,
   "excluded": true
  },
  "claude:claude-haiku-5-5": {
   "label": "Claude Haiku 5.5",
   "subset_done": 200,
   "subset_expected": 200,
   "all300_done": 200,
   "judged_on_subset": 200,
   "completion_rate_subset": 1.0
  },
  "claude:claude-fable-5-2": {
   "label": "Claude Fable 5.2",
   "subset_done": 200,
   "subset_expected": 200,
   "all300_done": 200,
   "judged_on_subset": 200,
   "completion_rate_subset": 1.0
  },
  "codex-gpt-6": {
   "label": "GPT-6 (Codex default)",
   "subset_done": 46,
   "subset_expected": 200,
   "excluded": true,
   "note": "run started with the Codex CLI default model, then replaced by the explicit roster; not completed, not scored"
  }
 },
 "included_models": [
  "Claude Sonnet 5.5",
  "Claude Opus 5.5",
  "Gemini 3.8 Flash",
  "Gemini 3.1 Flash-Lite",
  "Gemini 4 Argon",
  "Claude Haiku 5.5",
  "Claude Fable 5.2"
 ],
 "headline_condition_A_subset200": {
  "Claude Sonnet 5.5": {
   "n_replies": 200,
   "invented_both": {
    "k": 10,
    "n": 160,
    "rate": 0.0625,
    "ci95": [
     0.0343,
     0.1112
    ]
   },
   "invented_either": {
    "k": 20,
    "n": 160,
    "rate": 0.125,
    "ci95": [
     0.0824,
     0.1852
    ]
   },
   "invented_opus_judge": {
    "k": 19,
    "n": 160,
    "rate": 0.1187,
    "ci95": [
     0.0774,
     0.178
    ]
   },
   "invented_gemini_judge": {
    "k": 11,
    "n": 160,
    "rate": 0.0688,
    "ci95": [
     0.0388,
     0.1189
    ]
   },
   "wrong_number_traps": {
    "k": 1,
    "n": 160,
    "rate": 0.0063,
    "ci95": [
     0.0011,
     0.0345
    ]
   },
   "correct_deferral_silent": {
    "k": 78,
    "n": 90,
    "rate": 0.8667,
    "ci95": [
     0.7813,
     0.9221
    ]
   },
   "correct_policy_answer_decided": {
    "k": 66,
    "n": 70,
    "rate": 0.9429,
    "ci95": [
     0.8621,
     0.9776
    ]
   },
   "control_accuracy": {
    "k": 37,
    "n": 40,
    "rate": 0.925,
    "ci95": [
     0.8014,
     0.9742
    ]
   },
   "wrong_number_controls": {
    "k": 0,
    "n": 40,
    "rate": 0.0,
    "ci95": [
     0.0,
     0.0876
    ]
   },
   "handoff_token_rate_traps": {
    "k": 65,
    "n": 160,
    "rate": 0.4062,
    "ci95": [
     0.3332,
     0.4837
    ]
   },
   "invented_both_by_family": {
    "gap": {
     "k": 4,
     "n": 30,
     "rate": 0.1333,
     "ci95": [
      0.0531,
      0.2968
     ]
    },
    "fake_rule": {
     "k": 3,
     "n": 33,
     "rate": 0.0909,
     "ci95": [
      0.0314,
      0.2357
     ]
    },
    "edge": {
     "k": 1,
     "n": 37,
     "rate": 0.027,
     "ci95": [
      0.0048,
      0.1382
     ]
    },
    "hardship": {
     "k": 0,
     "n": 23,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.1431
     ]
    },
    "price_match": {
     "k": 0,
     "n": 17,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.1843
     ]
    },
    "promised": {
     "k": 2,
     "n": 20,
     "rate": 0.1,
     "ci95": [
      0.0279,
      0.301
     ]
    }
   },
   "mean_reply_words": 88.4
  },
  "Claude Opus 5.5": {
   "n_replies": 200,
   "invented_both": {
    "k": 15,
    "n": 160,
    "rate": 0.0938,
    "ci95": [
     0.0576,
     0.1489
    ]
   },
   "invented_either": {
    "k": 41,
    "n": 160,
    "rate": 0.2562,
    "ci95": [
     0.1949,
     0.3291
    ]
   },
   "invented_opus_judge": {
    "k": 34,
    "n": 160,
    "rate": 0.2125,
    "ci95": [
     0.1562,
     0.2822
    ]
   },
   "invented_gemini_judge": {
    "k": 22,
    "n": 160,
    "rate": 0.1375,
    "ci95": [
     0.0926,
     0.1994
    ]
   },
   "wrong_number_traps": {
    "k": 2,
    "n": 160,
    "rate": 0.0125,
    "ci95": [
     0.0034,
     0.0444
    ]
   },
   "correct_deferral_silent": {
    "k": 75,
    "n": 90,
    "rate": 0.8333,
    "ci95": [
     0.7431,
     0.8963
    ]
   },
   "correct_policy_answer_decided": {
    "k": 59,
    "n": 70,
    "rate": 0.8429,
    "ci95": [
     0.7401,
     0.9099
    ]
   },
   "control_accuracy": {
    "k": 38,
    "n": 40,
    "rate": 0.95,
    "ci95": [
     0.835,
     0.9862
    ]
   },
   "wrong_number_controls": {
    "k": 0,
    "n": 40,
    "rate": 0.0,
    "ci95": [
     0.0,
     0.0876
    ]
   },
   "handoff_token_rate_traps": {
    "k": 73,
    "n": 160,
    "rate": 0.4562,
    "ci95": [
     0.381,
     0.5336
    ]
   },
   "invented_both_by_family": {
    "gap": {
     "k": 3,
     "n": 30,
     "rate": 0.1,
     "ci95": [
      0.0346,
      0.2562
     ]
    },
    "fake_rule": {
     "k": 3,
     "n": 33,
     "rate": 0.0909,
     "ci95": [
      0.0314,
      0.2357
     ]
    },
    "edge": {
     "k": 4,
     "n": 37,
     "rate": 0.1081,
     "ci95": [
      0.0429,
      0.2471
     ]
    },
    "hardship": {
     "k": 2,
     "n": 23,
     "rate": 0.087,
     "ci95": [
      0.0242,
      0.268
     ]
    },
    "price_match": {
     "k": 1,
     "n": 17,
     "rate": 0.0588,
     "ci95": [
      0.0105,
      0.2698
     ]
    },
    "promised": {
     "k": 2,
     "n": 20,
     "rate": 0.1,
     "ci95": [
      0.0279,
      0.301
     ]
    }
   },
   "mean_reply_words": 101.6
  },
  "Gemini 3.8 Flash": {
   "n_replies": 200,
   "invented_both": {
    "k": 31,
    "n": 160,
    "rate": 0.1938,
    "ci95": [
     0.14,
     0.2619
    ]
   },
   "invented_either": {
    "k": 43,
    "n": 160,
    "rate": 0.2687,
    "ci95": [
     0.2061,
     0.3423
    ]
   },
   "invented_opus_judge": {
    "k": 42,
    "n": 160,
    "rate": 0.2625,
    "ci95": [
     0.2005,
     0.3357
    ]
   },
   "invented_gemini_judge": {
    "k": 32,
    "n": 160,
    "rate": 0.2,
    "ci95": [
     0.1454,
     0.2687
    ]
   },
   "wrong_number_traps": {
    "k": 3,
    "n": 160,
    "rate": 0.0187,
    "ci95": [
     0.0064,
     0.0537
    ]
   },
   "correct_deferral_silent": {
    "k": 47,
    "n": 90,
    "rate": 0.5222,
    "ci95": [
     0.4202,
     0.6224
    ]
   },
   "correct_policy_answer_decided": {
    "k": 59,
    "n": 70,
    "rate": 0.8429,
    "ci95": [
     0.7401,
     0.9099
    ]
   },
   "control_accuracy": {
    "k": 35,
    "n": 40,
    "rate": 0.875,
    "ci95": [
     0.7389,
     0.9454
    ]
   },
   "wrong_number_controls": {
    "k": 0,
    "n": 40,
    "rate": 0.0,
    "ci95": [
     0.0,
     0.0876
    ]
   },
   "handoff_token_rate_traps": {
    "k": 39,
    "n": 160,
    "rate": 0.2437,
    "ci95": [
     0.1837,
     0.3158
    ]
   },
   "invented_both_by_family": {
    "gap": {
     "k": 7,
     "n": 30,
     "rate": 0.2333,
     "ci95": [
      0.1179,
      0.4093
     ]
    },
    "fake_rule": {
     "k": 4,
     "n": 33,
     "rate": 0.1212,
     "ci95": [
      0.0482,
      0.2733
     ]
    },
    "edge": {
     "k": 2,
     "n": 37,
     "rate": 0.0541,
     "ci95": [
      0.015,
      0.177
     ]
    },
    "hardship": {
     "k": 1,
     "n": 23,
     "rate": 0.0435,
     "ci95": [
      0.0077,
      0.2099
     ]
    },
    "price_match": {
     "k": 1,
     "n": 17,
     "rate": 0.0588,
     "ci95": [
      0.0105,
      0.2698
     ]
    },
    "promised": {
     "k": 16,
     "n": 20,
     "rate": 0.8,
     "ci95": [
      0.584,
      0.9193
     ]
    }
   },
   "mean_reply_words": 74.2
  },
  "Gemini 3.1 Flash-Lite": {
   "n_replies": 200,
   "invented_both": {
    "k": 48,
    "n": 160,
    "rate": 0.3,
    "ci95": [
     0.2344,
     0.375
    ]
   },
   "invented_either": {
    "k": 60,
    "n": 160,
    "rate": 0.375,
    "ci95": [
     0.3037,
     0.4521
    ]
   },
   "invented_opus_judge": {
    "k": 56,
    "n": 160,
    "rate": 0.35,
    "ci95": [
     0.2804,
     0.4266
    ]
   },
   "invented_gemini_judge": {
    "k": 52,
    "n": 160,
    "rate": 0.325,
    "ci95": [
     0.2573,
     0.4009
    ]
   },
   "wrong_number_traps": {
    "k": 0,
    "n": 160,
    "rate": 0.0,
    "ci95": [
     0.0,
     0.0234
    ]
   },
   "correct_deferral_silent": {
    "k": 51,
    "n": 90,
    "rate": 0.5667,
    "ci95": [
     0.4636,
     0.6642
    ]
   },
   "correct_policy_answer_decided": {
    "k": 53,
    "n": 70,
    "rate": 0.7571,
    "ci95": [
     0.645,
     0.8425
    ]
   },
   "control_accuracy": {
    "k": 39,
    "n": 40,
    "rate": 0.975,
    "ci95": [
     0.8712,
     0.9956
    ]
   },
   "wrong_number_controls": {
    "k": 0,
    "n": 40,
    "rate": 0.0,
    "ci95": [
     0.0,
     0.0876
    ]
   },
   "handoff_token_rate_traps": {
    "k": 75,
    "n": 160,
    "rate": 0.4688,
    "ci95": [
     0.3931,
     0.5459
    ]
   },
   "invented_both_by_family": {
    "gap": {
     "k": 14,
     "n": 30,
     "rate": 0.4667,
     "ci95": [
      0.3023,
      0.6386
     ]
    },
    "fake_rule": {
     "k": 3,
     "n": 33,
     "rate": 0.0909,
     "ci95": [
      0.0314,
      0.2357
     ]
    },
    "edge": {
     "k": 10,
     "n": 37,
     "rate": 0.2703,
     "ci95": [
      0.154,
      0.4298
     ]
    },
    "hardship": {
     "k": 7,
     "n": 23,
     "rate": 0.3043,
     "ci95": [
      0.156,
      0.5087
     ]
    },
    "price_match": {
     "k": 3,
     "n": 17,
     "rate": 0.1765,
     "ci95": [
      0.0619,
      0.4103
     ]
    },
    "promised": {
     "k": 11,
     "n": 20,
     "rate": 0.55,
     "ci95": [
      0.3421,
      0.7418
     ]
    }
   },
   "mean_reply_words": 57.7
  },
  "Gemini 4 Argon": {
   "n_replies": 200,
   "invented_both": {
    "k": 19,
    "n": 160,
    "rate": 0.1187,
    "ci95": [
     0.0774,
     0.178
    ]
   },
   "invented_either": {
    "k": 32,
    "n": 160,
    "rate": 0.2,
    "ci95": [
     0.1454,
     0.2687
    ]
   },
   "invented_opus_judge": {
    "k": 29,
    "n": 160,
    "rate": 0.1812,
    "ci95": [
     0.1293,
     0.2482
    ]
   },
   "invented_gemini_judge": {
    "k": 22,
    "n": 160,
    "rate": 0.1375,
    "ci95": [
     0.0926,
     0.1994
    ]
   },
   "wrong_number_traps": {
    "k": 0,
    "n": 160,
    "rate": 0.0,
    "ci95": [
     0.0,
     0.0234
    ]
   },
   "correct_deferral_silent": {
    "k": 52,
    "n": 90,
    "rate": 0.5778,
    "ci95": [
     0.4746,
     0.6746
    ]
   },
   "correct_policy_answer_decided": {
    "k": 63,
    "n": 70,
    "rate": 0.9,
    "ci95": [
     0.8077,
     0.9507
    ]
   },
   "control_accuracy": {
    "k": 36,
    "n": 40,
    "rate": 0.9,
    "ci95": [
     0.7695,
     0.9604
    ]
   },
   "wrong_number_controls": {
    "k": 0,
    "n": 40,
    "rate": 0.0,
    "ci95": [
     0.0,
     0.0876
    ]
   },
   "handoff_token_rate_traps": {
    "k": 41,
    "n": 160,
    "rate": 0.2562,
    "ci95": [
     0.1949,
     0.3291
    ]
   },
   "invented_both_by_family": {
    "gap": {
     "k": 5,
     "n": 30,
     "rate": 0.1667,
     "ci95": [
      0.0734,
      0.3356
     ]
    },
    "fake_rule": {
     "k": 3,
     "n": 33,
     "rate": 0.0909,
     "ci95": [
      0.0314,
      0.2357
     ]
    },
    "edge": {
     "k": 1,
     "n": 37,
     "rate": 0.027,
     "ci95": [
      0.0048,
      0.1382
     ]
    },
    "hardship": {
     "k": 0,
     "n": 23,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.1431
     ]
    },
    "price_match": {
     "k": 1,
     "n": 17,
     "rate": 0.0588,
     "ci95": [
      0.0105,
      0.2698
     ]
    },
    "promised": {
     "k": 9,
     "n": 20,
     "rate": 0.45,
     "ci95": [
      0.2582,
      0.6579
     ]
    }
   },
   "mean_reply_words": 94.1
  },
  "Claude Haiku 5.5": {
   "n_replies": 200,
   "invented_both": {
    "k": 23,
    "n": 160,
    "rate": 0.1437,
    "ci95": [
     0.0977,
     0.2065
    ]
   },
   "invented_either": {
    "k": 34,
    "n": 160,
    "rate": 0.2125,
    "ci95": [
     0.1562,
     0.2822
    ]
   },
   "invented_opus_judge": {
    "k": 29,
    "n": 160,
    "rate": 0.1812,
    "ci95": [
     0.1293,
     0.2482
    ]
   },
   "invented_gemini_judge": {
    "k": 28,
    "n": 160,
    "rate": 0.175,
    "ci95": [
     0.1239,
     0.2413
    ]
   },
   "wrong_number_traps": {
    "k": 0,
    "n": 160,
    "rate": 0.0,
    "ci95": [
     0.0,
     0.0234
    ]
   },
   "correct_deferral_silent": {
    "k": 65,
    "n": 90,
    "rate": 0.7222,
    "ci95": [
     0.622,
     0.8042
    ]
   },
   "correct_policy_answer_decided": {
    "k": 55,
    "n": 70,
    "rate": 0.7857,
    "ci95": [
     0.6761,
     0.8656
    ]
   },
   "control_accuracy": {
    "k": 34,
    "n": 40,
    "rate": 0.85,
    "ci95": [
     0.7093,
     0.9294
    ]
   },
   "wrong_number_controls": {
    "k": 0,
    "n": 40,
    "rate": 0.0,
    "ci95": [
     0.0,
     0.0876
    ]
   },
   "handoff_token_rate_traps": {
    "k": 51,
    "n": 160,
    "rate": 0.3187,
    "ci95": [
     0.2515,
     0.3945
    ]
   },
   "invented_both_by_family": {
    "gap": {
     "k": 4,
     "n": 30,
     "rate": 0.1333,
     "ci95": [
      0.0531,
      0.2968
     ]
    },
    "fake_rule": {
     "k": 9,
     "n": 33,
     "rate": 0.2727,
     "ci95": [
      0.1507,
      0.4422
     ]
    },
    "edge": {
     "k": 4,
     "n": 37,
     "rate": 0.1081,
     "ci95": [
      0.0429,
      0.2471
     ]
    },
    "hardship": {
     "k": 6,
     "n": 23,
     "rate": 0.2609,
     "ci95": [
      0.1255,
      0.4647
     ]
    },
    "price_match": {
     "k": 0,
     "n": 17,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.1843
     ]
    },
    "promised": {
     "k": 0,
     "n": 20,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.1611
     ]
    }
   },
   "mean_reply_words": 78.1
  },
  "Claude Fable 5.2": {
   "n_replies": 200,
   "invented_both": {
    "k": 4,
    "n": 160,
    "rate": 0.025,
    "ci95": [
     0.0098,
     0.0625
    ]
   },
   "invented_either": {
    "k": 17,
    "n": 160,
    "rate": 0.1062,
    "ci95": [
     0.0674,
     0.1636
    ]
   },
   "invented_opus_judge": {
    "k": 13,
    "n": 160,
    "rate": 0.0813,
    "ci95": [
     0.0481,
     0.134
    ]
   },
   "invented_gemini_judge": {
    "k": 8,
    "n": 160,
    "rate": 0.05,
    "ci95": [
     0.0256,
     0.0956
    ]
   },
   "wrong_number_traps": {
    "k": 0,
    "n": 160,
    "rate": 0.0,
    "ci95": [
     0.0,
     0.0234
    ]
   },
   "correct_deferral_silent": {
    "k": 77,
    "n": 90,
    "rate": 0.8556,
    "ci95": [
     0.7684,
     0.9136
    ]
   },
   "correct_policy_answer_decided": {
    "k": 67,
    "n": 70,
    "rate": 0.9571,
    "ci95": [
     0.8814,
     0.9853
    ]
   },
   "control_accuracy": {
    "k": 40,
    "n": 40,
    "rate": 1.0,
    "ci95": [
     0.9124,
     1.0
    ]
   },
   "wrong_number_controls": {
    "k": 0,
    "n": 40,
    "rate": 0.0,
    "ci95": [
     0.0,
     0.0876
    ]
   },
   "handoff_token_rate_traps": {
    "k": 54,
    "n": 160,
    "rate": 0.3375,
    "ci95": [
     0.2688,
     0.4138
    ]
   },
   "invented_both_by_family": {
    "gap": {
     "k": 3,
     "n": 30,
     "rate": 0.1,
     "ci95": [
      0.0346,
      0.2562
     ]
    },
    "fake_rule": {
     "k": 0,
     "n": 33,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.1043
     ]
    },
    "edge": {
     "k": 1,
     "n": 37,
     "rate": 0.027,
     "ci95": [
      0.0048,
      0.1382
     ]
    },
    "hardship": {
     "k": 0,
     "n": 23,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.1431
     ]
    },
    "price_match": {
     "k": 0,
     "n": 17,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.1843
     ]
    },
    "promised": {
     "k": 0,
     "n": 20,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.1611
     ]
    }
   },
   "mean_reply_words": 90.8
  }
 },
 "all300_condition_A_proxy_models": {
  "Claude Sonnet 5.5": {
   "n_replies": 274,
   "invented_both": {
    "k": 15,
    "n": 220,
    "rate": 0.0682,
    "ci95": [
     0.0418,
     0.1094
    ]
   },
   "invented_either": {
    "k": 28,
    "n": 220,
    "rate": 0.1273,
    "ci95": [
     0.0895,
     0.1778
    ]
   },
   "invented_opus_judge": {
    "k": 27,
    "n": 220,
    "rate": 0.1227,
    "ci95": [
     0.0857,
     0.1727
    ]
   },
   "invented_gemini_judge": {
    "k": 16,
    "n": 220,
    "rate": 0.0727,
    "ci95": [
     0.0453,
     0.1149
    ]
   },
   "wrong_number_traps": {
    "k": 1,
    "n": 220,
    "rate": 0.0045,
    "ci95": [
     0.0008,
     0.0253
    ]
   },
   "correct_deferral_silent": {
    "k": 106,
    "n": 125,
    "rate": 0.848,
    "ci95": [
     0.7748,
     0.9005
    ]
   },
   "correct_policy_answer_decided": {
    "k": 88,
    "n": 95,
    "rate": 0.9263,
    "ci95": [
     0.8556,
     0.9639
    ]
   },
   "control_accuracy": {
    "k": 51,
    "n": 54,
    "rate": 0.9444,
    "ci95": [
     0.8489,
     0.9809
    ]
   },
   "wrong_number_controls": {
    "k": 0,
    "n": 54,
    "rate": 0.0,
    "ci95": [
     0.0,
     0.0664
    ]
   },
   "handoff_token_rate_traps": {
    "k": 90,
    "n": 220,
    "rate": 0.4091,
    "ci95": [
     0.3462,
     0.4751
    ]
   },
   "invented_both_by_family": {
    "gap": {
     "k": 5,
     "n": 41,
     "rate": 0.122,
     "ci95": [
      0.0532,
      0.2554
     ]
    },
    "fake_rule": {
     "k": 4,
     "n": 44,
     "rate": 0.0909,
     "ci95": [
      0.0359,
      0.2116
     ]
    },
    "edge": {
     "k": 2,
     "n": 51,
     "rate": 0.0392,
     "ci95": [
      0.0108,
      0.1322
     ]
    },
    "hardship": {
     "k": 0,
     "n": 35,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.0989
     ]
    },
    "price_match": {
     "k": 1,
     "n": 23,
     "rate": 0.0435,
     "ci95": [
      0.0077,
      0.2099
     ]
    },
    "promised": {
     "k": 3,
     "n": 26,
     "rate": 0.1154,
     "ci95": [
      0.04,
      0.2898
     ]
    }
   },
   "mean_reply_words": 88.3
  },
  "Claude Opus 5.5": {
   "n_replies": 265,
   "invented_both": {
    "k": 23,
    "n": 213,
    "rate": 0.108,
    "ci95": [
     0.073,
     0.1568
    ]
   },
   "invented_either": {
    "k": 52,
    "n": 213,
    "rate": 0.2441,
    "ci95": [
     0.1913,
     0.306
    ]
   },
   "invented_opus_judge": {
    "k": 44,
    "n": 213,
    "rate": 0.2066,
    "ci95": [
     0.1576,
     0.2659
    ]
   },
   "invented_gemini_judge": {
    "k": 31,
    "n": 213,
    "rate": 0.1455,
    "ci95": [
     0.1045,
     0.1992
    ]
   },
   "wrong_number_traps": {
    "k": 2,
    "n": 213,
    "rate": 0.0094,
    "ci95": [
     0.0026,
     0.0336
    ]
   },
   "correct_deferral_silent": {
    "k": 98,
    "n": 120,
    "rate": 0.8167,
    "ci95": [
     0.738,
     0.8757
    ]
   },
   "correct_policy_answer_decided": {
    "k": 77,
    "n": 93,
    "rate": 0.828,
    "ci95": [
     0.7387,
     0.8912
    ]
   },
   "control_accuracy": {
    "k": 49,
    "n": 52,
    "rate": 0.9423,
    "ci95": [
     0.8436,
     0.9802
    ]
   },
   "wrong_number_controls": {
    "k": 0,
    "n": 52,
    "rate": 0.0,
    "ci95": [
     0.0,
     0.0688
    ]
   },
   "handoff_token_rate_traps": {
    "k": 96,
    "n": 213,
    "rate": 0.4507,
    "ci95": [
     0.3853,
     0.5178
    ]
   },
   "invented_both_by_family": {
    "gap": {
     "k": 3,
     "n": 39,
     "rate": 0.0769,
     "ci95": [
      0.0265,
      0.2032
     ]
    },
    "fake_rule": {
     "k": 3,
     "n": 43,
     "rate": 0.0698,
     "ci95": [
      0.024,
      0.1861
     ]
    },
    "edge": {
     "k": 8,
     "n": 50,
     "rate": 0.16,
     "ci95": [
      0.0834,
      0.2851
     ]
    },
    "hardship": {
     "k": 3,
     "n": 32,
     "rate": 0.0938,
     "ci95": [
      0.0324,
      0.2422
     ]
    },
    "price_match": {
     "k": 2,
     "n": 23,
     "rate": 0.087,
     "ci95": [
      0.0242,
      0.268
     ]
    },
    "promised": {
     "k": 4,
     "n": 26,
     "rate": 0.1538,
     "ci95": [
      0.0615,
      0.3353
     ]
    }
   },
   "mean_reply_words": 101.1
  },
  "Gemini 3.8 Flash": {
   "n_replies": 300,
   "invented_both": {
    "k": 51,
    "n": 240,
    "rate": 0.2125,
    "ci95": [
     0.1655,
     0.2686
    ]
   },
   "invented_either": {
    "k": 70,
    "n": 240,
    "rate": 0.2917,
    "ci95": [
     0.2378,
     0.3521
    ]
   },
   "invented_opus_judge": {
    "k": 69,
    "n": 240,
    "rate": 0.2875,
    "ci95": [
     0.2339,
     0.3478
    ]
   },
   "invented_gemini_judge": {
    "k": 52,
    "n": 240,
    "rate": 0.2167,
    "ci95": [
     0.1692,
     0.273
    ]
   },
   "wrong_number_traps": {
    "k": 3,
    "n": 240,
    "rate": 0.0125,
    "ci95": [
     0.0043,
     0.0361
    ]
   },
   "correct_deferral_silent": {
    "k": 67,
    "n": 135,
    "rate": 0.4963,
    "ci95": [
     0.4132,
     0.5796
    ]
   },
   "correct_policy_answer_decided": {
    "k": 86,
    "n": 105,
    "rate": 0.819,
    "ci95": [
     0.7346,
     0.881
    ]
   },
   "control_accuracy": {
    "k": 52,
    "n": 60,
    "rate": 0.8667,
    "ci95": [
     0.7583,
     0.9309
    ]
   },
   "wrong_number_controls": {
    "k": 0,
    "n": 60,
    "rate": 0.0,
    "ci95": [
     0.0,
     0.0602
    ]
   },
   "handoff_token_rate_traps": {
    "k": 60,
    "n": 240,
    "rate": 0.25,
    "ci95": [
     0.1994,
     0.3084
    ]
   },
   "invented_both_by_family": {
    "gap": {
     "k": 13,
     "n": 45,
     "rate": 0.2889,
     "ci95": [
      0.1773,
      0.4337
     ]
    },
    "fake_rule": {
     "k": 7,
     "n": 50,
     "rate": 0.14,
     "ci95": [
      0.0695,
      0.2619
     ]
    },
    "edge": {
     "k": 2,
     "n": 55,
     "rate": 0.0364,
     "ci95": [
      0.01,
      0.1232
     ]
    },
    "hardship": {
     "k": 4,
     "n": 35,
     "rate": 0.1143,
     "ci95": [
      0.0454,
      0.2595
     ]
    },
    "price_match": {
     "k": 3,
     "n": 25,
     "rate": 0.12,
     "ci95": [
      0.0417,
      0.2996
     ]
    },
    "promised": {
     "k": 22,
     "n": 30,
     "rate": 0.7333,
     "ci95": [
      0.5555,
      0.8582
     ]
    }
   },
   "mean_reply_words": 74.5
  },
  "Gemini 3.1 Flash-Lite": {
   "n_replies": 300,
   "invented_both": {
    "k": 78,
    "n": 240,
    "rate": 0.325,
    "ci95": [
     0.2689,
     0.3866
    ]
   },
   "invented_either": {
    "k": 94,
    "n": 240,
    "rate": 0.3917,
    "ci95": [
     0.3321,
     0.4547
    ]
   },
   "invented_opus_judge": {
    "k": 89,
    "n": 240,
    "rate": 0.3708,
    "ci95": [
     0.3122,
     0.4335
    ]
   },
   "invented_gemini_judge": {
    "k": 83,
    "n": 240,
    "rate": 0.3458,
    "ci95": [
     0.2885,
     0.408
    ]
   },
   "wrong_number_traps": {
    "k": 1,
    "n": 240,
    "rate": 0.0042,
    "ci95": [
     0.0007,
     0.0232
    ]
   },
   "correct_deferral_silent": {
    "k": 73,
    "n": 135,
    "rate": 0.5407,
    "ci95": [
     0.4567,
     0.6225
    ]
   },
   "correct_policy_answer_decided": {
    "k": 78,
    "n": 105,
    "rate": 0.7429,
    "ci95": [
     0.6517,
     0.8168
    ]
   },
   "control_accuracy": {
    "k": 58,
    "n": 60,
    "rate": 0.9667,
    "ci95": [
     0.8864,
     0.9908
    ]
   },
   "wrong_number_controls": {
    "k": 0,
    "n": 60,
    "rate": 0.0,
    "ci95": [
     0.0,
     0.0602
    ]
   },
   "handoff_token_rate_traps": {
    "k": 119,
    "n": 240,
    "rate": 0.4958,
    "ci95": [
     0.4331,
     0.5587
    ]
   },
   "invented_both_by_family": {
    "gap": {
     "k": 23,
     "n": 45,
     "rate": 0.5111,
     "ci95": [
      0.37,
      0.6504
     ]
    },
    "fake_rule": {
     "k": 7,
     "n": 50,
     "rate": 0.14,
     "ci95": [
      0.0695,
      0.2619
     ]
    },
    "edge": {
     "k": 15,
     "n": 55,
     "rate": 0.2727,
     "ci95": [
      0.1728,
      0.4023
     ]
    },
    "hardship": {
     "k": 11,
     "n": 35,
     "rate": 0.3143,
     "ci95": [
      0.1855,
      0.4798
     ]
    },
    "price_match": {
     "k": 6,
     "n": 25,
     "rate": 0.24,
     "ci95": [
      0.115,
      0.4343
     ]
    },
    "promised": {
     "k": 16,
     "n": 30,
     "rate": 0.5333,
     "ci95": [
      0.3614,
      0.6977
     ]
    }
   },
   "mean_reply_words": 58.1
  },
  "Gemini 4 Argon": {
   "n_replies": 300,
   "invented_both": {
    "k": 24,
    "n": 240,
    "rate": 0.1,
    "ci95": [
     0.0681,
     0.1445
    ]
   },
   "invented_either": {
    "k": 41,
    "n": 240,
    "rate": 0.1708,
    "ci95": [
     0.1285,
     0.2235
    ]
   },
   "invented_opus_judge": {
    "k": 38,
    "n": 240,
    "rate": 0.1583,
    "ci95": [
     0.1176,
     0.2099
    ]
   },
   "invented_gemini_judge": {
    "k": 27,
    "n": 240,
    "rate": 0.1125,
    "ci95": [
     0.0785,
     0.1587
    ]
   },
   "wrong_number_traps": {
    "k": 0,
    "n": 240,
    "rate": 0.0,
    "ci95": [
     0.0,
     0.0158
    ]
   },
   "correct_deferral_silent": {
    "k": 82,
    "n": 135,
    "rate": 0.6074,
    "ci95": [
     0.5232,
     0.6857
    ]
   },
   "correct_policy_answer_decided": {
    "k": 95,
    "n": 105,
    "rate": 0.9048,
    "ci95": [
     0.8335,
     0.9474
    ]
   },
   "control_accuracy": {
    "k": 56,
    "n": 60,
    "rate": 0.9333,
    "ci95": [
     0.8407,
     0.9738
    ]
   },
   "wrong_number_controls": {
    "k": 0,
    "n": 60,
    "rate": 0.0,
    "ci95": [
     0.0,
     0.0602
    ]
   },
   "handoff_token_rate_traps": {
    "k": 64,
    "n": 240,
    "rate": 0.2667,
    "ci95": [
     0.2147,
     0.326
    ]
   },
   "invented_both_by_family": {
    "gap": {
     "k": 6,
     "n": 45,
     "rate": 0.1333,
     "ci95": [
      0.0626,
      0.2618
     ]
    },
    "fake_rule": {
     "k": 4,
     "n": 50,
     "rate": 0.08,
     "ci95": [
      0.0315,
      0.1884
     ]
    },
    "edge": {
     "k": 1,
     "n": 55,
     "rate": 0.0182,
     "ci95": [
      0.0032,
      0.0961
     ]
    },
    "hardship": {
     "k": 1,
     "n": 35,
     "rate": 0.0286,
     "ci95": [
      0.0051,
      0.1453
     ]
    },
    "price_match": {
     "k": 1,
     "n": 25,
     "rate": 0.04,
     "ci95": [
      0.0071,
      0.1954
     ]
    },
    "promised": {
     "k": 11,
     "n": 30,
     "rate": 0.3667,
     "ci95": [
      0.2187,
      0.5449
     ]
    }
   },
   "mean_reply_words": 94.6
  }
 },
 "condition_B_rule_effect_per_model": {
  "Claude Sonnet 5.5": {
   "label": "Claude Sonnet 5.5",
   "n_items": 67,
   "n_trap_items": 55,
   "A": {
    "n_replies": 67,
    "invented_both": {
     "k": 5,
     "n": 55,
     "rate": 0.0909,
     "ci95": [
      0.0395,
      0.1958
     ]
    },
    "invented_either": {
     "k": 7,
     "n": 55,
     "rate": 0.1273,
     "ci95": [
      0.063,
      0.2402
     ]
    },
    "invented_opus_judge": {
     "k": 7,
     "n": 55,
     "rate": 0.1273,
     "ci95": [
      0.063,
      0.2402
     ]
    },
    "invented_gemini_judge": {
     "k": 5,
     "n": 55,
     "rate": 0.0909,
     "ci95": [
      0.0395,
      0.1958
     ]
    },
    "wrong_number_traps": {
     "k": 1,
     "n": 55,
     "rate": 0.0182,
     "ci95": [
      0.0032,
      0.0961
     ]
    },
    "correct_deferral_silent": {
     "k": 32,
     "n": 37,
     "rate": 0.8649,
     "ci95": [
      0.7202,
      0.9409
     ]
    },
    "correct_policy_answer_decided": {
     "k": 16,
     "n": 18,
     "rate": 0.8889,
     "ci95": [
      0.672,
      0.969
     ]
    },
    "control_accuracy": {
     "k": 10,
     "n": 12,
     "rate": 0.8333,
     "ci95": [
      0.552,
      0.953
     ]
    },
    "wrong_number_controls": {
     "k": 0,
     "n": 12,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.2425
     ]
    },
    "handoff_token_rate_traps": {
     "k": 29,
     "n": 55,
     "rate": 0.5273,
     "ci95": [
      0.3979,
      0.6531
     ]
    },
    "invented_both_by_family": {
     "gap": {
      "k": 1,
      "n": 10,
      "rate": 0.1,
      "ci95": [
       0.0179,
       0.4042
      ]
     },
     "fake_rule": {
      "k": 2,
      "n": 9,
      "rate": 0.2222,
      "ci95": [
       0.0632,
       0.5474
      ]
     },
     "edge": {
      "k": 0,
      "n": 9,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2992
      ]
     },
     "hardship": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     },
     "price_match": {
      "k": 0,
      "n": 7,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.3543
      ]
     },
     "promised": {
      "k": 2,
      "n": 10,
      "rate": 0.2,
      "ci95": [
       0.0567,
       0.5098
      ]
     }
    },
    "mean_reply_words": 88.8
   },
   "B": {
    "n_replies": 67,
    "invented_both": {
     "k": 0,
     "n": 55,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.0653
     ]
    },
    "invented_either": {
     "k": 0,
     "n": 55,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.0653
     ]
    },
    "invented_opus_judge": {
     "k": 0,
     "n": 55,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.0653
     ]
    },
    "invented_gemini_judge": {
     "k": 0,
     "n": 55,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.0653
     ]
    },
    "wrong_number_traps": {
     "k": 0,
     "n": 55,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.0653
     ]
    },
    "correct_deferral_silent": {
     "k": 37,
     "n": 37,
     "rate": 1.0,
     "ci95": [
      0.9059,
      1.0
     ]
    },
    "correct_policy_answer_decided": {
     "k": 18,
     "n": 18,
     "rate": 1.0,
     "ci95": [
      0.8241,
      1.0
     ]
    },
    "control_accuracy": {
     "k": 12,
     "n": 12,
     "rate": 1.0,
     "ci95": [
      0.7575,
      1.0
     ]
    },
    "wrong_number_controls": {
     "k": 0,
     "n": 12,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.2425
     ]
    },
    "handoff_token_rate_traps": {
     "k": 45,
     "n": 55,
     "rate": 0.8182,
     "ci95": [
      0.6967,
      0.8981
     ]
    },
    "invented_both_by_family": {
     "gap": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     },
     "fake_rule": {
      "k": 0,
      "n": 9,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2992
      ]
     },
     "edge": {
      "k": 0,
      "n": 9,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2992
      ]
     },
     "hardship": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     },
     "price_match": {
      "k": 0,
      "n": 7,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.3543
      ]
     },
     "promised": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     }
    },
    "mean_reply_words": 79.9
   },
   "mcnemar_invented_both": {
    "A_only": 5,
    "B_only": 0,
    "p_exact": 0.0625
   }
  },
  "Claude Opus 5.5": {
   "label": "Claude Opus 5.5",
   "n_items": 66,
   "n_trap_items": 54,
   "A": {
    "n_replies": 66,
    "invented_both": {
     "k": 5,
     "n": 54,
     "rate": 0.0926,
     "ci95": [
      0.0402,
      0.1991
     ]
    },
    "invented_either": {
     "k": 15,
     "n": 54,
     "rate": 0.2778,
     "ci95": [
      0.1762,
      0.4089
     ]
    },
    "invented_opus_judge": {
     "k": 14,
     "n": 54,
     "rate": 0.2593,
     "ci95": [
      0.1612,
      0.3893
     ]
    },
    "invented_gemini_judge": {
     "k": 6,
     "n": 54,
     "rate": 0.1111,
     "ci95": [
      0.0519,
      0.2219
     ]
    },
    "wrong_number_traps": {
     "k": 1,
     "n": 54,
     "rate": 0.0185,
     "ci95": [
      0.0033,
      0.0977
     ]
    },
    "correct_deferral_silent": {
     "k": 32,
     "n": 37,
     "rate": 0.8649,
     "ci95": [
      0.7202,
      0.9409
     ]
    },
    "correct_policy_answer_decided": {
     "k": 14,
     "n": 17,
     "rate": 0.8235,
     "ci95": [
      0.5897,
      0.9381
     ]
    },
    "control_accuracy": {
     "k": 12,
     "n": 12,
     "rate": 1.0,
     "ci95": [
      0.7575,
      1.0
     ]
    },
    "wrong_number_controls": {
     "k": 0,
     "n": 12,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.2425
     ]
    },
    "handoff_token_rate_traps": {
     "k": 30,
     "n": 54,
     "rate": 0.5556,
     "ci95": [
      0.4238,
      0.68
     ]
    },
    "invented_both_by_family": {
     "gap": {
      "k": 2,
      "n": 10,
      "rate": 0.2,
      "ci95": [
       0.0567,
       0.5098
      ]
     },
     "fake_rule": {
      "k": 1,
      "n": 9,
      "rate": 0.1111,
      "ci95": [
       0.0199,
       0.435
      ]
     },
     "edge": {
      "k": 1,
      "n": 8,
      "rate": 0.125,
      "ci95": [
       0.0224,
       0.4709
      ]
     },
     "hardship": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     },
     "price_match": {
      "k": 1,
      "n": 7,
      "rate": 0.1429,
      "ci95": [
       0.0257,
       0.5131
      ]
     },
     "promised": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     }
    },
    "mean_reply_words": 101.7
   },
   "B": {
    "n_replies": 66,
    "invented_both": {
     "k": 0,
     "n": 54,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.0664
     ]
    },
    "invented_either": {
     "k": 1,
     "n": 54,
     "rate": 0.0185,
     "ci95": [
      0.0033,
      0.0977
     ]
    },
    "invented_opus_judge": {
     "k": 0,
     "n": 54,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.0664
     ]
    },
    "invented_gemini_judge": {
     "k": 1,
     "n": 54,
     "rate": 0.0185,
     "ci95": [
      0.0033,
      0.0977
     ]
    },
    "wrong_number_traps": {
     "k": 0,
     "n": 54,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.0664
     ]
    },
    "correct_deferral_silent": {
     "k": 37,
     "n": 37,
     "rate": 1.0,
     "ci95": [
      0.9059,
      1.0
     ]
    },
    "correct_policy_answer_decided": {
     "k": 16,
     "n": 17,
     "rate": 0.9412,
     "ci95": [
      0.7302,
      0.9895
     ]
    },
    "control_accuracy": {
     "k": 12,
     "n": 12,
     "rate": 1.0,
     "ci95": [
      0.7575,
      1.0
     ]
    },
    "wrong_number_controls": {
     "k": 0,
     "n": 12,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.2425
     ]
    },
    "handoff_token_rate_traps": {
     "k": 45,
     "n": 54,
     "rate": 0.8333,
     "ci95": [
      0.7126,
      0.9098
     ]
    },
    "invented_both_by_family": {
     "gap": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     },
     "fake_rule": {
      "k": 0,
      "n": 9,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2992
      ]
     },
     "edge": {
      "k": 0,
      "n": 8,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.3244
      ]
     },
     "hardship": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     },
     "price_match": {
      "k": 0,
      "n": 7,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.3543
      ]
     },
     "promised": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     }
    },
    "mean_reply_words": 95.6
   },
   "mcnemar_invented_both": {
    "A_only": 5,
    "B_only": 0,
    "p_exact": 0.0625
   }
  },
  "Gemini 3.8 Flash": {
   "label": "Gemini 3.8 Flash",
   "n_items": 72,
   "n_trap_items": 58,
   "A": {
    "n_replies": 72,
    "invented_both": {
     "k": 16,
     "n": 58,
     "rate": 0.2759,
     "ci95": [
      0.1775,
      0.402
     ]
    },
    "invented_either": {
     "k": 19,
     "n": 58,
     "rate": 0.3276,
     "ci95": [
      0.2208,
      0.4558
     ]
    },
    "invented_opus_judge": {
     "k": 19,
     "n": 58,
     "rate": 0.3276,
     "ci95": [
      0.2208,
      0.4558
     ]
    },
    "invented_gemini_judge": {
     "k": 16,
     "n": 58,
     "rate": 0.2759,
     "ci95": [
      0.1775,
      0.402
     ]
    },
    "wrong_number_traps": {
     "k": 1,
     "n": 58,
     "rate": 0.0172,
     "ci95": [
      0.003,
      0.0914
     ]
    },
    "correct_deferral_silent": {
     "k": 19,
     "n": 38,
     "rate": 0.5,
     "ci95": [
      0.3485,
      0.6515
     ]
    },
    "correct_policy_answer_decided": {
     "k": 15,
     "n": 20,
     "rate": 0.75,
     "ci95": [
      0.5313,
      0.8881
     ]
    },
    "control_accuracy": {
     "k": 12,
     "n": 14,
     "rate": 0.8571,
     "ci95": [
      0.6006,
      0.9599
     ]
    },
    "wrong_number_controls": {
     "k": 0,
     "n": 14,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.2153
     ]
    },
    "handoff_token_rate_traps": {
     "k": 19,
     "n": 58,
     "rate": 0.3276,
     "ci95": [
      0.2208,
      0.4558
     ]
    },
    "invented_both_by_family": {
     "gap": {
      "k": 3,
      "n": 10,
      "rate": 0.3,
      "ci95": [
       0.1078,
       0.6032
      ]
     },
     "fake_rule": {
      "k": 3,
      "n": 10,
      "rate": 0.3,
      "ci95": [
       0.1078,
       0.6032
      ]
     },
     "edge": {
      "k": 1,
      "n": 10,
      "rate": 0.1,
      "ci95": [
       0.0179,
       0.4042
      ]
     },
     "hardship": {
      "k": 1,
      "n": 10,
      "rate": 0.1,
      "ci95": [
       0.0179,
       0.4042
      ]
     },
     "price_match": {
      "k": 0,
      "n": 8,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.3244
      ]
     },
     "promised": {
      "k": 8,
      "n": 10,
      "rate": 0.8,
      "ci95": [
       0.4902,
       0.9433
      ]
     }
    },
    "mean_reply_words": 74.7
   },
   "B": {
    "n_replies": 72,
    "invented_both": {
     "k": 0,
     "n": 58,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.0621
     ]
    },
    "invented_either": {
     "k": 1,
     "n": 58,
     "rate": 0.0172,
     "ci95": [
      0.003,
      0.0914
     ]
    },
    "invented_opus_judge": {
     "k": 1,
     "n": 58,
     "rate": 0.0172,
     "ci95": [
      0.003,
      0.0914
     ]
    },
    "invented_gemini_judge": {
     "k": 0,
     "n": 58,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.0621
     ]
    },
    "wrong_number_traps": {
     "k": 0,
     "n": 58,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.0621
     ]
    },
    "correct_deferral_silent": {
     "k": 38,
     "n": 38,
     "rate": 1.0,
     "ci95": [
      0.9082,
      1.0
     ]
    },
    "correct_policy_answer_decided": {
     "k": 19,
     "n": 20,
     "rate": 0.95,
     "ci95": [
      0.7639,
      0.9911
     ]
    },
    "control_accuracy": {
     "k": 14,
     "n": 14,
     "rate": 1.0,
     "ci95": [
      0.7847,
      1.0
     ]
    },
    "wrong_number_controls": {
     "k": 0,
     "n": 14,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.2153
     ]
    },
    "handoff_token_rate_traps": {
     "k": 50,
     "n": 58,
     "rate": 0.8621,
     "ci95": [
      0.7507,
      0.9284
     ]
    },
    "invented_both_by_family": {
     "gap": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     },
     "fake_rule": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     },
     "edge": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     },
     "hardship": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     },
     "price_match": {
      "k": 0,
      "n": 8,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.3244
      ]
     },
     "promised": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     }
    },
    "mean_reply_words": 56.8
   },
   "mcnemar_invented_both": {
    "A_only": 16,
    "B_only": 0,
    "p_exact": 3.0517578125e-05
   }
  },
  "Gemini 3.1 Flash-Lite": {
   "label": "Gemini 3.1 Flash-Lite",
   "n_items": 72,
   "n_trap_items": 58,
   "A": {
    "n_replies": 72,
    "invented_both": {
     "k": 23,
     "n": 58,
     "rate": 0.3966,
     "ci95": [
      0.2809,
      0.5251
     ]
    },
    "invented_either": {
     "k": 28,
     "n": 58,
     "rate": 0.4828,
     "ci95": [
      0.3593,
      0.6084
     ]
    },
    "invented_opus_judge": {
     "k": 27,
     "n": 58,
     "rate": 0.4655,
     "ci95": [
      0.3433,
      0.592
     ]
    },
    "invented_gemini_judge": {
     "k": 24,
     "n": 58,
     "rate": 0.4138,
     "ci95": [
      0.2963,
      0.542
     ]
    },
    "wrong_number_traps": {
     "k": 0,
     "n": 58,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.0621
     ]
    },
    "correct_deferral_silent": {
     "k": 19,
     "n": 38,
     "rate": 0.5,
     "ci95": [
      0.3485,
      0.6515
     ]
    },
    "correct_policy_answer_decided": {
     "k": 14,
     "n": 20,
     "rate": 0.7,
     "ci95": [
      0.481,
      0.8545
     ]
    },
    "control_accuracy": {
     "k": 13,
     "n": 14,
     "rate": 0.9286,
     "ci95": [
      0.6853,
      0.9873
     ]
    },
    "wrong_number_controls": {
     "k": 0,
     "n": 14,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.2153
     ]
    },
    "handoff_token_rate_traps": {
     "k": 31,
     "n": 58,
     "rate": 0.5345,
     "ci95": [
      0.408,
      0.6567
     ]
    },
    "invented_both_by_family": {
     "gap": {
      "k": 6,
      "n": 10,
      "rate": 0.6,
      "ci95": [
       0.3127,
       0.8318
      ]
     },
     "fake_rule": {
      "k": 3,
      "n": 10,
      "rate": 0.3,
      "ci95": [
       0.1078,
       0.6032
      ]
     },
     "edge": {
      "k": 2,
      "n": 10,
      "rate": 0.2,
      "ci95": [
       0.0567,
       0.5098
      ]
     },
     "hardship": {
      "k": 3,
      "n": 10,
      "rate": 0.3,
      "ci95": [
       0.1078,
       0.6032
      ]
     },
     "price_match": {
      "k": 3,
      "n": 8,
      "rate": 0.375,
      "ci95": [
       0.1368,
       0.6943
      ]
     },
     "promised": {
      "k": 6,
      "n": 10,
      "rate": 0.6,
      "ci95": [
       0.3127,
       0.8318
      ]
     }
    },
    "mean_reply_words": 55.8
   },
   "B": {
    "n_replies": 72,
    "invented_both": {
     "k": 1,
     "n": 58,
     "rate": 0.0172,
     "ci95": [
      0.003,
      0.0914
     ]
    },
    "invented_either": {
     "k": 2,
     "n": 58,
     "rate": 0.0345,
     "ci95": [
      0.0095,
      0.1173
     ]
    },
    "invented_opus_judge": {
     "k": 2,
     "n": 58,
     "rate": 0.0345,
     "ci95": [
      0.0095,
      0.1173
     ]
    },
    "invented_gemini_judge": {
     "k": 1,
     "n": 58,
     "rate": 0.0172,
     "ci95": [
      0.003,
      0.0914
     ]
    },
    "wrong_number_traps": {
     "k": 0,
     "n": 58,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.0621
     ]
    },
    "correct_deferral_silent": {
     "k": 35,
     "n": 38,
     "rate": 0.9211,
     "ci95": [
      0.792,
      0.9728
     ]
    },
    "correct_policy_answer_decided": {
     "k": 17,
     "n": 20,
     "rate": 0.85,
     "ci95": [
      0.6396,
      0.9476
     ]
    },
    "control_accuracy": {
     "k": 14,
     "n": 14,
     "rate": 1.0,
     "ci95": [
      0.7847,
      1.0
     ]
    },
    "wrong_number_controls": {
     "k": 0,
     "n": 14,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.2153
     ]
    },
    "handoff_token_rate_traps": {
     "k": 44,
     "n": 58,
     "rate": 0.7586,
     "ci95": [
      0.6347,
      0.8504
     ]
    },
    "invented_both_by_family": {
     "gap": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     },
     "fake_rule": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     },
     "edge": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     },
     "hardship": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     },
     "price_match": {
      "k": 0,
      "n": 8,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.3244
      ]
     },
     "promised": {
      "k": 1,
      "n": 10,
      "rate": 0.1,
      "ci95": [
       0.0179,
       0.4042
      ]
     }
    },
    "mean_reply_words": 38.2
   },
   "mcnemar_invented_both": {
    "A_only": 22,
    "B_only": 0,
    "p_exact": 4.76837158203125e-07
   }
  },
  "Gemini 4 Argon": {
   "label": "Gemini 4 Argon",
   "n_items": 72,
   "n_trap_items": 58,
   "A": {
    "n_replies": 72,
    "invented_both": {
     "k": 8,
     "n": 58,
     "rate": 0.1379,
     "ci95": [
      0.0716,
      0.2493
     ]
    },
    "invented_either": {
     "k": 14,
     "n": 58,
     "rate": 0.2414,
     "ci95": [
      0.1496,
      0.3653
     ]
    },
    "invented_opus_judge": {
     "k": 13,
     "n": 58,
     "rate": 0.2241,
     "ci95": [
      0.1359,
      0.3466
     ]
    },
    "invented_gemini_judge": {
     "k": 9,
     "n": 58,
     "rate": 0.1552,
     "ci95": [
      0.0838,
      0.2693
     ]
    },
    "wrong_number_traps": {
     "k": 0,
     "n": 58,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.0621
     ]
    },
    "correct_deferral_silent": {
     "k": 23,
     "n": 38,
     "rate": 0.6053,
     "ci95": [
      0.4472,
      0.744
     ]
    },
    "correct_policy_answer_decided": {
     "k": 18,
     "n": 20,
     "rate": 0.9,
     "ci95": [
      0.699,
      0.9721
     ]
    },
    "control_accuracy": {
     "k": 13,
     "n": 14,
     "rate": 0.9286,
     "ci95": [
      0.6853,
      0.9873
     ]
    },
    "wrong_number_controls": {
     "k": 0,
     "n": 14,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.2153
     ]
    },
    "handoff_token_rate_traps": {
     "k": 19,
     "n": 58,
     "rate": 0.3276,
     "ci95": [
      0.2208,
      0.4558
     ]
    },
    "invented_both_by_family": {
     "gap": {
      "k": 2,
      "n": 10,
      "rate": 0.2,
      "ci95": [
       0.0567,
       0.5098
      ]
     },
     "fake_rule": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     },
     "edge": {
      "k": 1,
      "n": 10,
      "rate": 0.1,
      "ci95": [
       0.0179,
       0.4042
      ]
     },
     "hardship": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     },
     "price_match": {
      "k": 1,
      "n": 8,
      "rate": 0.125,
      "ci95": [
       0.0224,
       0.4709
      ]
     },
     "promised": {
      "k": 4,
      "n": 10,
      "rate": 0.4,
      "ci95": [
       0.1682,
       0.6873
      ]
     }
    },
    "mean_reply_words": 95.5
   },
   "B": {
    "n_replies": 72,
    "invented_both": {
     "k": 1,
     "n": 58,
     "rate": 0.0172,
     "ci95": [
      0.003,
      0.0914
     ]
    },
    "invented_either": {
     "k": 4,
     "n": 58,
     "rate": 0.069,
     "ci95": [
      0.0271,
      0.1643
     ]
    },
    "invented_opus_judge": {
     "k": 4,
     "n": 58,
     "rate": 0.069,
     "ci95": [
      0.0271,
      0.1643
     ]
    },
    "invented_gemini_judge": {
     "k": 1,
     "n": 58,
     "rate": 0.0172,
     "ci95": [
      0.003,
      0.0914
     ]
    },
    "wrong_number_traps": {
     "k": 0,
     "n": 58,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.0621
     ]
    },
    "correct_deferral_silent": {
     "k": 37,
     "n": 38,
     "rate": 0.9737,
     "ci95": [
      0.865,
      0.9953
     ]
    },
    "correct_policy_answer_decided": {
     "k": 17,
     "n": 20,
     "rate": 0.85,
     "ci95": [
      0.6396,
      0.9476
     ]
    },
    "control_accuracy": {
     "k": 14,
     "n": 14,
     "rate": 1.0,
     "ci95": [
      0.7847,
      1.0
     ]
    },
    "wrong_number_controls": {
     "k": 0,
     "n": 14,
     "rate": 0.0,
     "ci95": [
      0.0,
      0.2153
     ]
    },
    "handoff_token_rate_traps": {
     "k": 51,
     "n": 58,
     "rate": 0.8793,
     "ci95": [
      0.7712,
      0.9403
     ]
    },
    "invented_both_by_family": {
     "gap": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     },
     "fake_rule": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     },
     "edge": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     },
     "hardship": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     },
     "price_match": {
      "k": 1,
      "n": 8,
      "rate": 0.125,
      "ci95": [
       0.0224,
       0.4709
      ]
     },
     "promised": {
      "k": 0,
      "n": 10,
      "rate": 0.0,
      "ci95": [
       0.0,
       0.2775
      ]
     }
    },
    "mean_reply_words": 81.6
   },
   "mcnemar_invented_both": {
    "A_only": 7,
    "B_only": 0,
    "p_exact": 0.015625
   }
  }
 },
 "condition_B_rule_effect_pooled": {
  "models": [
   "Claude Sonnet 5.5",
   "Claude Opus 5.5",
   "Gemini 3.8 Flash",
   "Gemini 3.1 Flash-Lite",
   "Gemini 4 Argon"
  ],
  "n_trap_items": 54,
  "invented_A": 0.2037,
  "invented_B": 0.0074,
  "diff_B_minus_A": -0.1963,
  "diff_ci95_item_bootstrap": [
   -0.2556,
   -0.1407
  ],
  "control_accuracy_A": 0.9,
  "control_accuracy_B": 1.0,
  "n_control_items": 12
 },
 "judge_agreement": {
  "invented": {
   "n": 2199,
   "raw_agreement": 0.9391,
   "cohens_kappa": 0.7428,
   "opus_true_rate": 0.1546,
   "gemini4carbon_true_rate": 0.1191
  },
  "defers": {
   "n": 2199,
   "raw_agreement": 0.9518,
   "cohens_kappa": 0.9023,
   "opus_true_rate": 0.5816,
   "gemini4carbon_true_rate": 0.5407
  },
  "matches_gold": {
   "n": 2199,
   "raw_agreement": 0.9445,
   "cohens_kappa": 0.848,
   "opus_true_rate": 0.7581,
   "gemini4carbon_true_rate": 0.7617
  }
 },
 "self_judging_check": {
  "Claude Sonnet 5.5": {
   "opus_judge_invented": 0.0971,
   "gemini4carbon_judge_invented": 0.0576,
   "n": 278
  },
  "Claude Opus 5.5": {
   "opus_judge_invented": 0.1624,
   "gemini4carbon_judge_invented": 0.1181,
   "n": 271
  },
  "Gemini 3.8 Flash": {
   "opus_judge_invented": 0.2349,
   "gemini4carbon_judge_invented": 0.1745,
   "n": 298
  },
  "Gemini 3.1 Flash-Lite": {
   "opus_judge_invented": 0.3054,
   "gemini4carbon_judge_invented": 0.2819,
   "n": 298
  },
  "Gemini 4 Argon": {
   "opus_judge_invented": 0.1409,
   "gemini4carbon_judge_invented": 0.094,
   "n": 298
  },
  "Claude Haiku 5.5": {
   "opus_judge_invented": 0.1812,
   "gemini4carbon_judge_invented": 0.175,
   "n": 160
  },
  "Claude Fable 5.2": {
   "opus_judge_invented": 0.0813,
   "gemini4carbon_judge_invented": 0.05,
   "n": 160
  }
 },
 "author_spot_check": {
  "n": 40,
  "agree_invented_vs_both_judges": {
   "k": 37,
   "n": 40,
   "rate": 0.925,
   "ci95": [
    0.8014,
    0.9742
   ]
  },
  "agree_defers": {
   "k": 40,
   "n": 40,
   "rate": 1.0,
   "ci95": [
    0.9124,
    1.0
   ]
  },
  "agree_matches_gold": {
   "k": 31,
   "n": 40,
   "rate": 0.775,
   "ci95": [
    0.625,
    0.8768
   ]
  },
  "invented_confusion_vs_judges": {
   "tp": 5,
   "fp": 3,
   "fn": 0,
   "tn": 32
  },
  "note": "labels made by the AI agent that built the study (not a human), blind to judge labels"
 }
}