From e91dc43876620b52644daad38fd4c4ac371b4fb3 Mon Sep 17 00:00:00 2001 From: Dang Nguyen Date: Wed, 24 Jun 2026 17:41:03 -0500 Subject: [PATCH 1/2] Add cluster-bootstrap 95% CIs for the perturbation recall tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ci_recall.py reports the §5 recall tables with 95% bootstrap CIs over papers (resampling unit = paper, pooled recall = sum(detected)/sum(injected)). Point estimates match the perturbation scorer. Also gitignores generated figures under perturbation/plots/. Split out of the AUC CI PR. Co-Authored-By: Claude Opus 4.8 --- benchmarks/perturbation/.gitignore | 3 + benchmarks/perturbation/ci_recall.py | 129 +++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 benchmarks/perturbation/ci_recall.py diff --git a/benchmarks/perturbation/.gitignore b/benchmarks/perturbation/.gitignore index 5cce8cb..175d040 100644 --- a/benchmarks/perturbation/.gitignore +++ b/benchmarks/perturbation/.gitignore @@ -20,3 +20,6 @@ __pycache__/ .ipynb_checkpoints/ *.pdf + +# Generated figures +plots/ diff --git a/benchmarks/perturbation/ci_recall.py b/benchmarks/perturbation/ci_recall.py new file mode 100644 index 0000000..0a48cf5 --- /dev/null +++ b/benchmarks/perturbation/ci_recall.py @@ -0,0 +1,129 @@ +"""Cluster-bootstrap-over-papers 95% CIs for the §5 recall tables. + +Resampling unit = paper. For each (method, model[, category]) cell we collect +per-paper (detected, injected) counts over the 24-paper frontier subset (the +same papers/ptypes as _combine_gpt_claude.py / configs/submission/subset_bigmodels_*), +then bootstrap-resample the contributing papers with replacement, recompute +pooled recall = sum(detected)/sum(injected), and take 2.5/97.5 percentiles. + +Point estimate matches the paper (e.g., GPT-5.5 progressive = 571/797 = 71.6%). +No significance tests — CIs only. +""" +import json +import numpy as np +from pathlib import Path + +ROOT = Path(__file__).resolve().parent / "results" +SCORE = "llm_t4_grounded" +B = 5000 +RNG = np.random.default_rng(42) + +# (results_dir, [papers], [ptypes]) — frontier 24-paper subset. +DOMAINS = [ + ("cs_CC_scaleup_v2", ["paper_001", "paper_003", "paper_004"], ["surface", "claim_theoretical", "logic"]), + ("full_cs_LG", ["paper_001", "paper_002", "paper_003"], ["surface", "statement_empirical", "experimental"]), + ("full_econ_EM", ["paper_001", "paper_002", "paper_003"], ["surface", "statement_empirical", "experimental"]), + ("full_hep_ex", ["paper_001", "paper_002", "paper_003"], ["surface", "statement_empirical", "experimental"]), + ("full_math_all", ["paper_001", "paper_002", "paper_004"], ["surface", "statement_empirical", "experimental"]), + ("full_physics_atm_clus", ["paper_001", "paper_002", "paper_003"], ["surface", "statement_empirical", "experimental"]), + ("full_q_bio_GN", ["paper_001", "paper_002", "paper_003"], ["surface", "statement_empirical", "experimental"]), + ("full_stat_AP", ["paper_001", "paper_002", "paper_003"], ["surface", "statement_empirical", "experimental"]), +] + +# ptype -> high-level category for tab:recall-by-type +CAT = { + "surface": "Surface", + "statement_empirical": "Claim", + "claim_theoretical": "Claim", + "logic": "Reasoning", + "experimental": "Experimental", +} + +MODELS = { + "GPT-5.5": "gpt-5.5", "Claude-Opus-4.7": "claude-opus-4.7", + "Grok-4.1-Fast": "grok-4.1-fast", "DeepSeek-V4-Flash": "deepseek-v4-flash", + "Qwen3.6-35B-A3B": "qwen3.6-35b-a3b", "Gemini-3.1-Flash-Lite": "gemini-3.1-flash-lite-preview", +} + + +def base_of(domain): + return domain.replace("full_", "").replace("_scaleup_v2", "") + + +def load(domain, model, ptype, method, paper): + if method == "reviewer3": + p = ROOT / f"full_{base_of(domain)}_reviewer3" / "reviewer3" / ptype / "reviewer3" / paper / "score" / SCORE / f"{paper}_score.json" + else: + p = ROOT / domain / model / ptype / method / paper / "score" / SCORE / f"{paper}_score.json" + if not p.exists(): + return None + return json.loads(p.read_text()) + + +def per_paper_counts(model_slug, method, category=None): + """Return list of (detected, injected) per paper for this cell.""" + rows = [] + for domain, papers, ptypes in DOMAINS: + for paper in papers: + det = inj = 0 + got = False + for pt in ptypes: + if category and CAT.get(pt) != category: + continue + s = load(domain, model_slug, pt, method, paper) + if s is None: + continue + got = True + det += s["n_detected"] + inj += s["n_injected"] + if got and inj > 0: + rows.append((det, inj)) + return rows + + +def boot_ci(rows): + if not rows: + return (float("nan"), float("nan"), float("nan"), 0, 0) + det = np.array([r[0] for r in rows], float) + inj = np.array([r[1] for r in rows], float) + point = det.sum() / inj.sum() + n = len(rows) + idx = RNG.integers(0, n, size=(B, n)) + bd = det[idx].sum(axis=1) + bi = inj[idx].sum(axis=1) + rec = bd / bi + lo, hi = np.percentile(rec, [2.5, 97.5]) + return (point, lo, hi, int(det.sum()), int(inj.sum())) + + +def fmt(point, lo, hi, d, i): + return f"{point*100:5.1f}% [{lo*100:4.1f}, {hi*100:4.1f}] ({d}/{i})" + + +if __name__ == "__main__": + print("=== tab:recall-overall (per model × method), frontier subset ===") + for label, slug in MODELS.items(): + line = f"{label:22s}" + for method in ("zero_shot", "coarse", "progressive"): + rows = per_paper_counts(slug, method) + line += " | " + fmt(*boot_ci(rows)) + print(line) + r3 = per_paper_counts("reviewer3", "reviewer3") + print("Reviewer3 (overall) | " + fmt(*boot_ci(r3))) + + print("\n=== tab:recall-by-type (best backend per system) ===") + cells = [ + ("coarse / DeepSeek-V4", "deepseek-v4-flash", "coarse"), + ("zero-shot / GPT-5.5", "gpt-5.5", "zero_shot"), + ("OpenAIReview / GPT-5.5","gpt-5.5", "progressive"), + ("Reviewer3", "reviewer3", "reviewer3"), + ] + cats = [None, "Experimental", "Claim", "Reasoning", "Surface"] + print(f"{'cell':24s} | " + " | ".join((c or 'Overall') for c in cats)) + for name, slug, method in cells: + line = f"{name:24s}" + for c in cats: + rows = per_paper_counts(slug, method, c) + p, lo, hi, d, i = boot_ci(rows) + line += f" | {p*100:4.1f}[{lo*100:.1f},{hi*100:.1f}]n{len(rows)}" + print(line) From 1ed5eb531419b127399c9f85a564191b8cb0e539 Mon Sep 17 00:00:00 2001 From: Dang Nguyen Date: Fri, 26 Jun 2026 21:04:57 -0500 Subject: [PATCH 2/2] Make recall CIs config- and test-driven Move paper sets, result dirs, and table specs into a local JSON config (--config), matching the conference-study ci_auc.py pattern. Split the pooled-recall + cluster-bootstrap math into pure paper_rows/boot_ci helpers and cover them in tests/test_ci_recall.py (in-memory counts, no fixtures). Empty cells render as NO DATA, and an unknown table kind errors out instead of raising deep in a renderer. Output is unchanged on real data. Co-Authored-By: Claude Opus 4.8 (1M context) --- benchmarks/perturbation/ci_recall.py | 238 ++++++++++++++++----------- tests/test_ci_recall.py | 96 +++++++++++ 2 files changed, 239 insertions(+), 95 deletions(-) create mode 100644 tests/test_ci_recall.py diff --git a/benchmarks/perturbation/ci_recall.py b/benchmarks/perturbation/ci_recall.py index 0a48cf5..d6f5cc2 100644 --- a/benchmarks/perturbation/ci_recall.py +++ b/benchmarks/perturbation/ci_recall.py @@ -1,87 +1,68 @@ -"""Cluster-bootstrap-over-papers 95% CIs for the §5 recall tables. - -Resampling unit = paper. For each (method, model[, category]) cell we collect -per-paper (detected, injected) counts over the 24-paper frontier subset (the -same papers/ptypes as _combine_gpt_claude.py / configs/submission/subset_bigmodels_*), -then bootstrap-resample the contributing papers with replacement, recompute -pooled recall = sum(detected)/sum(injected), and take 2.5/97.5 percentiles. - -Point estimate matches the paper (e.g., GPT-5.5 progressive = 571/797 = 71.6%). -No significance tests — CIs only. +"""95% confidence intervals for the recall tables in the paper. + +Recall here is the fraction of injected errors that a review system detects. The +intervals come from a cluster bootstrap, where the cluster is a paper: collect every +paper's (detected, injected) error counts for a given method and model, resample those +papers with replacement many times, recompute the pooled recall (total detected divided +by total injected) on each resample, and take the 2.5th and 97.5th percentiles. +Resampling whole papers makes the interval reflect how much the estimate would move if +the paper sample changed. Pass an error category to restrict a cell to one error type. + +The paper sets, result directories, and tables to print live in a JSON config (--config, +defaults to ci_recall_tables.json), kept local because it names internal result +directories and model ids. The bootstrap math is covered by tests/test_ci_recall.py on +in-memory counts. Two table kinds: by_model (recall per model and method) and by_category +(recall split by error category for selected systems). + +Every cell draws from one seeded random generator, consumed in config order. A table's +CIs reproduce exactly only when the whole config runs unchanged, so adding or reordering +cells shifts the intervals of later ones. """ import json -import numpy as np from pathlib import Path -ROOT = Path(__file__).resolve().parent / "results" -SCORE = "llm_t4_grounded" +import numpy as np + B = 5000 RNG = np.random.default_rng(42) -# (results_dir, [papers], [ptypes]) — frontier 24-paper subset. -DOMAINS = [ - ("cs_CC_scaleup_v2", ["paper_001", "paper_003", "paper_004"], ["surface", "claim_theoretical", "logic"]), - ("full_cs_LG", ["paper_001", "paper_002", "paper_003"], ["surface", "statement_empirical", "experimental"]), - ("full_econ_EM", ["paper_001", "paper_002", "paper_003"], ["surface", "statement_empirical", "experimental"]), - ("full_hep_ex", ["paper_001", "paper_002", "paper_003"], ["surface", "statement_empirical", "experimental"]), - ("full_math_all", ["paper_001", "paper_002", "paper_004"], ["surface", "statement_empirical", "experimental"]), - ("full_physics_atm_clus", ["paper_001", "paper_002", "paper_003"], ["surface", "statement_empirical", "experimental"]), - ("full_q_bio_GN", ["paper_001", "paper_002", "paper_003"], ["surface", "statement_empirical", "experimental"]), - ("full_stat_AP", ["paper_001", "paper_002", "paper_003"], ["surface", "statement_empirical", "experimental"]), -] - -# ptype -> high-level category for tab:recall-by-type -CAT = { - "surface": "Surface", - "statement_empirical": "Claim", - "claim_theoretical": "Claim", - "logic": "Reasoning", - "experimental": "Experimental", -} - -MODELS = { - "GPT-5.5": "gpt-5.5", "Claude-Opus-4.7": "claude-opus-4.7", - "Grok-4.1-Fast": "grok-4.1-fast", "DeepSeek-V4-Flash": "deepseek-v4-flash", - "Qwen3.6-35B-A3B": "qwen3.6-35b-a3b", "Gemini-3.1-Flash-Lite": "gemini-3.1-flash-lite-preview", -} - -def base_of(domain): - return domain.replace("full_", "").replace("_scaleup_v2", "") +# ---- pure aggregation + bootstrap (unit-tested in tests/test_ci_recall.py) ---- +def paper_rows(per_paper, category_of=None, category=None): + """Per-paper (detected, injected) counts for one table cell. -def load(domain, model, ptype, method, paper): - if method == "reviewer3": - p = ROOT / f"full_{base_of(domain)}_reviewer3" / "reviewer3" / ptype / "reviewer3" / paper / "score" / SCORE / f"{paper}_score.json" - else: - p = ROOT / domain / model / ptype / method / paper / "score" / SCORE / f"{paper}_score.json" - if not p.exists(): - return None - return json.loads(p.read_text()) - - -def per_paper_counts(model_slug, method, category=None): - """Return list of (detected, injected) per paper for this cell.""" + `per_paper` maps a paper id to its per-error-type counts, each a + (detected, injected) pair. With `category` set, only error types whose + `category_of[error_type]` equals it are summed, restricting the cell to one + category (e.g. Surface). A paper is dropped when it contributes no matching + error type or zero injected errors. + """ + category_of = category_of or {} rows = [] - for domain, papers, ptypes in DOMAINS: - for paper in papers: - det = inj = 0 - got = False - for pt in ptypes: - if category and CAT.get(pt) != category: - continue - s = load(domain, model_slug, pt, method, paper) - if s is None: - continue - got = True - det += s["n_detected"] - inj += s["n_injected"] - if got and inj > 0: - rows.append((det, inj)) + for counts_by_type in per_paper.values(): + detected = injected = 0 + matched = False + for error_type, (n_detected, n_injected) in counts_by_type.items(): + if category and category_of.get(error_type) != category: + continue + detected += n_detected + injected += n_injected + matched = True + if matched and injected > 0: + rows.append((detected, injected)) return rows def boot_ci(rows): + """Pooled recall and its 95% cluster-bootstrap CI for one cell. + + `rows` is the per-paper (detected, injected) list from paper_rows. Returns + (point, lo, hi, detected, injected): the pooled recall + sum(detected) / sum(injected), the 2.5/97.5 bootstrap percentiles over + paper resamples, and the summed counts. All-nan with zeroed counts when + `rows` is empty. + """ if not rows: return (float("nan"), float("nan"), float("nan"), 0, 0) det = np.array([r[0] for r in rows], float) @@ -89,41 +70,108 @@ def boot_ci(rows): point = det.sum() / inj.sum() n = len(rows) idx = RNG.integers(0, n, size=(B, n)) - bd = det[idx].sum(axis=1) - bi = inj[idx].sum(axis=1) - rec = bd / bi + rec = det[idx].sum(axis=1) / inj[idx].sum(axis=1) lo, hi = np.percentile(rec, [2.5, 97.5]) return (point, lo, hi, int(det.sum()), int(inj.sum())) +# ---- reading score files off disk ---- + +def base_of(domain): + return domain.replace("full_", "").replace("_scaleup_v2", "") + + +def load_score(root, score, domain, model, etype, method, paper): + if method == "reviewer3": + p = root / f"full_{base_of(domain)}_reviewer3" / "reviewer3" / etype / "reviewer3" / paper / "score" / score / f"{paper}_score.json" + else: + p = root / domain / model / etype / method / paper / "score" / score / f"{paper}_score.json" + if not p.exists(): + return None + return json.loads(p.read_text()) + + +def load_cell(config, root, model_slug, method): + """Read each paper's per-error-type (detected, injected) counts for a cell. + + Returns {paper_id: {error_type: (detected, injected)}} over the domains in + the config, ready for paper_rows. Each (domain, paper) is its own cluster, + so the same paper id under different domains stays separate. + """ + per_paper = {} + for dom in config["domains"]: + domain = dom["dir"] + for paper in dom["papers"]: + counts = {} + for etype in dom["error_types"]: + s = load_score(root, config["score"], domain, model_slug, etype, method, paper) + if s is None: + continue + counts[etype] = (s["n_detected"], s["n_injected"]) + if counts: + per_paper[f"{domain}/{paper}"] = counts + return per_paper + + +def cell_ci(config, root, slug, method, category=None): + """(boot_ci tuple, n_papers) for one (model, method[, category]) cell.""" + rows = paper_rows(load_cell(config, root, slug, method), config.get("categories"), category) + return boot_ci(rows), len(rows) + + +# ---- table renderers ---- + def fmt(point, lo, hi, d, i): + if i == 0: # no injected errors loaded: empty category or a misconfigured cell + return f"{'NO DATA':^30}" return f"{point*100:5.1f}% [{lo*100:4.1f}, {hi*100:4.1f}] ({d}/{i})" -if __name__ == "__main__": - print("=== tab:recall-overall (per model × method), frontier subset ===") - for label, slug in MODELS.items(): +def run_by_model(config, root, table): + """Recall per model x method, plus any single-cell extra rows (e.g. Reviewer3).""" + methods = table["methods"] + for label, slug in table["models"].items(): line = f"{label:22s}" - for method in ("zero_shot", "coarse", "progressive"): - rows = per_paper_counts(slug, method) - line += " | " + fmt(*boot_ci(rows)) + for method in methods: + ci, _ = cell_ci(config, root, slug, method) + line += " | " + fmt(*ci) print(line) - r3 = per_paper_counts("reviewer3", "reviewer3") - print("Reviewer3 (overall) | " + fmt(*boot_ci(r3))) - - print("\n=== tab:recall-by-type (best backend per system) ===") - cells = [ - ("coarse / DeepSeek-V4", "deepseek-v4-flash", "coarse"), - ("zero-shot / GPT-5.5", "gpt-5.5", "zero_shot"), - ("OpenAIReview / GPT-5.5","gpt-5.5", "progressive"), - ("Reviewer3", "reviewer3", "reviewer3"), - ] - cats = [None, "Experimental", "Claim", "Reasoning", "Surface"] - print(f"{'cell':24s} | " + " | ".join((c or 'Overall') for c in cats)) - for name, slug, method in cells: - line = f"{name:24s}" + for extra in table.get("extra_rows", []): + ci, _ = cell_ci(config, root, extra["slug"], extra["method"]) + print(f"{extra['label']:22s}| " + fmt(*ci)) + + +def run_by_category(config, root, table): + """Recall split by error category for selected (system) cells.""" + cats = table["categories"] + print(f"{'cell':24s} | " + " | ".join((c or "Overall") for c in cats)) + for cell in table["cells"]: + line = f"{cell['label']:24s}" for c in cats: - rows = per_paper_counts(slug, method, c) - p, lo, hi, d, i = boot_ci(rows) - line += f" | {p*100:4.1f}[{lo*100:.1f},{hi*100:.1f}]n{len(rows)}" + (p, lo, hi, d, i), n = cell_ci(config, root, cell["slug"], cell["method"], c) + line += " | " + ("NO DATA" if n == 0 else f"{p*100:4.1f}[{lo*100:.1f},{hi*100:.1f}]n{n}") print(line) + + +DISPATCH = {"by_model": run_by_model, "by_category": run_by_category} + + +if __name__ == "__main__": + import argparse + + HERE = Path(__file__).resolve().parent + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--config", type=Path, default=HERE / "ci_recall_tables.json", + help="JSON defining domains, error-type categories, and tables. " + "Paths inside it are relative to the config's directory. " + "Defaults to ci_recall_tables.json.") + args = ap.parse_args() + config = json.loads(args.config.read_text()) + root = args.config.resolve().parent / config.get("results_dir", "results") + for table in config["tables"]: + kind = table["kind"] + if kind not in DISPATCH: + ap.error(f"unknown table kind {kind!r}; choices: {list(DISPATCH)}") + print(f"\n=== {table['title']} ===") + DISPATCH[kind](config, root, table) diff --git a/tests/test_ci_recall.py b/tests/test_ci_recall.py new file mode 100644 index 0000000..7689e29 --- /dev/null +++ b/tests/test_ci_recall.py @@ -0,0 +1,96 @@ +"""Unit tests for the perturbation recall + bootstrap-CI helpers (ci_recall.py). + +Uses tiny in-memory (detected, injected) counts (no result files), so the math +is checkable by hand: papers whose per-paper recall is a round fraction, and a +degenerate case where every paper has the same recall so the CI collapses onto +the point estimate. +""" + +import math +import sys +from pathlib import Path + +import numpy as np +import pytest + +_PERT = Path(__file__).resolve().parents[1] / "benchmarks" / "perturbation" +if str(_PERT) not in sys.path: + sys.path.insert(0, str(_PERT)) + +import ci_recall # noqa: E402 +from ci_recall import DISPATCH, boot_ci, paper_rows # noqa: E402 + + +CATEGORIES = {"surface": "Surface", "logic": "Reasoning"} + + +def _per_paper(): + """Two papers, each with surface + logic error types. + + p1: surface 1/2 + logic 2/2 -> 3 detected of 4 injected. + p2: surface 0/2 only -> 0 detected of 2 injected. + """ + return { + "d/p1": {"surface": (1, 2), "logic": (2, 2)}, + "d/p2": {"surface": (0, 2)}, + } + + +# ---- paper_rows: per-paper aggregation and category filtering ---- + +def test_paper_rows_sums_error_types_per_paper(): + assert paper_rows(_per_paper()) == [(3, 4), (0, 2)] + + +def test_paper_rows_filters_by_category(): + # Surface keeps both papers' surface counts. + assert paper_rows(_per_paper(), CATEGORIES, "Surface") == [(1, 2), (0, 2)] + # Reasoning (logic) exists only on p1; p2 is dropped. + assert paper_rows(_per_paper(), CATEGORIES, "Reasoning") == [(2, 2)] + + +def test_paper_rows_drops_zero_injected_paper(): + per_paper = {"d/p1": {"surface": (1, 2)}, "d/p3": {"surface": (0, 0)}} + assert paper_rows(per_paper) == [(1, 2)] + + +# ---- boot_ci: pooled recall point estimate and CI ---- + +def test_boot_ci_point_estimate_and_counts(): + point, lo, hi, det, inj = boot_ci([(3, 4), (0, 2)]) + assert point == 0.5 and det == 3 and inj == 6 + assert 0.0 <= lo <= point <= hi <= 1.0 + + +def test_boot_ci_pins_exact_bounds_under_fixed_seed(): + # Reseed the module generator so the resampling is reproducible, then pin the + # exact percentile bounds for papers with genuine between-paper variance. This + # checks the percentile core itself, which the inequality assertions above and + # the degenerate case below cannot. + saved = ci_recall.RNG + ci_recall.RNG = np.random.default_rng(0) + try: + point, lo, hi, detected, injected = boot_ci([(1, 2), (3, 4), (0, 5)]) + finally: + ci_recall.RNG = saved + assert (detected, injected) == (4, 11) + assert point == pytest.approx(4 / 11) + assert lo == pytest.approx(0.0) + assert hi == pytest.approx(0.75) + assert lo < point < hi # a real spread, not a collapsed interval + + +def test_boot_ci_degenerate_equal_recall_collapses_ci(): + # Every paper has recall 1/2, so every resample pools to 1/2 exactly. + point, lo, hi, det, inj = boot_ci([(1, 2), (3, 6)]) + assert point == 0.5 and lo == 0.5 and hi == 0.5 + + +def test_boot_ci_empty_is_nan(): + point, lo, hi, det, inj = boot_ci([]) + assert math.isnan(point) and math.isnan(lo) and math.isnan(hi) + assert det == 0 and inj == 0 + + +def test_dispatch_covers_the_two_kinds(): + assert set(DISPATCH) == {"by_model", "by_category"}