Add cluster-bootstrap 95% CIs for the perturbation recall tables

benchmarks/perturbation/.gitignore

@@ -20,3 +20,6 @@ __pycache__/
 .ipynb_checkpoints/
 
 *.pdf
+
+# Generated figures
+plots/

benchmarks/perturbation/ci_recall.py

@@ -0,0 +1,177 @@
+"""95% confidence intervals for the recall tables in the paper.
+
+Recall here is the fraction of injected errors that a review system detects. The
+intervals come from a cluster bootstrap, where the cluster is a paper: collect every
+paper's (detected, injected) error counts for a given method and model, resample those
+papers with replacement many times, recompute the pooled recall (total detected divided
+by total injected) on each resample, and take the 2.5th and 97.5th percentiles.
+Resampling whole papers makes the interval reflect how much the estimate would move if
+the paper sample changed. Pass an error category to restrict a cell to one error type.
+
+The paper sets, result directories, and tables to print live in a JSON config (--config,
+defaults to ci_recall_tables.json), kept local because it names internal result
+directories and model ids. The bootstrap math is covered by tests/test_ci_recall.py on
+in-memory counts. Two table kinds: by_model (recall per model and method) and by_category
+(recall split by error category for selected systems).
+
+Every cell draws from one seeded random generator, consumed in config order. A table's
+CIs reproduce exactly only when the whole config runs unchanged, so adding or reordering
+cells shifts the intervals of later ones.
+"""
+import json
+from pathlib import Path
+
+import numpy as np
+
+B = 5000
+RNG = np.random.default_rng(42)
+
+
+# ---- pure aggregation + bootstrap (unit-tested in tests/test_ci_recall.py) ----
+
+def paper_rows(per_paper, category_of=None, category=None):
+    """Per-paper (detected, injected) counts for one table cell.
+
+    `per_paper` maps a paper id to its per-error-type counts, each a
+    (detected, injected) pair. With `category` set, only error types whose
+    `category_of[error_type]` equals it are summed, restricting the cell to one
+    category (e.g. Surface). A paper is dropped when it contributes no matching
+    error type or zero injected errors.
+    """
+    category_of = category_of or {}
+    rows = []
+    for counts_by_type in per_paper.values():
+        detected = injected = 0
+        matched = False
+        for error_type, (n_detected, n_injected) in counts_by_type.items():
+            if category and category_of.get(error_type) != category:
+                continue
+            detected += n_detected
+            injected += n_injected
+            matched = True
+        if matched and injected > 0:
+            rows.append((detected, injected))
+    return rows
+
+
+def boot_ci(rows):
+    """Pooled recall and its 95% cluster-bootstrap CI for one cell.
+
+    `rows` is the per-paper (detected, injected) list from paper_rows. Returns
+    (point, lo, hi, detected, injected): the pooled recall
+    sum(detected) / sum(injected), the 2.5/97.5 bootstrap percentiles over
+    paper resamples, and the summed counts. All-nan with zeroed counts when
+    `rows` is empty.
+    """
+    if not rows:
+        return (float("nan"), float("nan"), float("nan"), 0, 0)
+    det = np.array([r[0] for r in rows], float)
+    inj = np.array([r[1] for r in rows], float)
+    point = det.sum() / inj.sum()
+    n = len(rows)
+    idx = RNG.integers(0, n, size=(B, n))
+    rec = det[idx].sum(axis=1) / inj[idx].sum(axis=1)
+    lo, hi = np.percentile(rec, [2.5, 97.5])
+    return (point, lo, hi, int(det.sum()), int(inj.sum()))
+
+
+# ---- reading score files off disk ----
+
+def base_of(domain):
+    return domain.replace("full_", "").replace("_scaleup_v2", "")
+
+
+def load_score(root, score, domain, model, etype, method, paper):
+    if method == "reviewer3":
+        p = root / f"full_{base_of(domain)}_reviewer3" / "reviewer3" / etype / "reviewer3" / paper / "score" / score / f"{paper}_score.json"
+    else:
+        p = root / domain / model / etype / method / paper / "score" / score / f"{paper}_score.json"
+    if not p.exists():
+        return None
+    return json.loads(p.read_text())
+
+
+def load_cell(config, root, model_slug, method):
+    """Read each paper's per-error-type (detected, injected) counts for a cell.
+
+    Returns {paper_id: {error_type: (detected, injected)}} over the domains in
+    the config, ready for paper_rows. Each (domain, paper) is its own cluster,
+    so the same paper id under different domains stays separate.
+    """
+    per_paper = {}
+    for dom in config["domains"]:
+        domain = dom["dir"]
+        for paper in dom["papers"]:
+            counts = {}
+            for etype in dom["error_types"]:
+                s = load_score(root, config["score"], domain, model_slug, etype, method, paper)
+                if s is None:
+                    continue
+                counts[etype] = (s["n_detected"], s["n_injected"])
+            if counts:
+                per_paper[f"{domain}/{paper}"] = counts
+    return per_paper
+
+
+def cell_ci(config, root, slug, method, category=None):
+    """(boot_ci tuple, n_papers) for one (model, method[, category]) cell."""
+    rows = paper_rows(load_cell(config, root, slug, method), config.get("categories"), category)
+    return boot_ci(rows), len(rows)
+
+
+# ---- table renderers ----
+
+def fmt(point, lo, hi, d, i):
+    if i == 0:  # no injected errors loaded: empty category or a misconfigured cell
+        return f"{'NO DATA':^30}"
+    return f"{point*100:5.1f}% [{lo*100:4.1f}, {hi*100:4.1f}]  ({d}/{i})"
+
+
+def run_by_model(config, root, table):
+    """Recall per model x method, plus any single-cell extra rows (e.g. Reviewer3)."""
+    methods = table["methods"]
+    for label, slug in table["models"].items():
+        line = f"{label:22s}"
+        for method in methods:
+            ci, _ = cell_ci(config, root, slug, method)
+            line += " | " + fmt(*ci)
+        print(line)
+    for extra in table.get("extra_rows", []):
+        ci, _ = cell_ci(config, root, extra["slug"], extra["method"])
+        print(f"{extra['label']:22s}| " + fmt(*ci))
+
+
+def run_by_category(config, root, table):
+    """Recall split by error category for selected (system) cells."""
+    cats = table["categories"]
+    print(f"{'cell':24s} | " + " | ".join((c or "Overall") for c in cats))
+    for cell in table["cells"]:
+        line = f"{cell['label']:24s}"
+        for c in cats:
+            (p, lo, hi, d, i), n = cell_ci(config, root, cell["slug"], cell["method"], c)
+            line += " | " + ("NO DATA" if n == 0 else f"{p*100:4.1f}[{lo*100:.1f},{hi*100:.1f}]n{n}")
+        print(line)
+
+
+DISPATCH = {"by_model": run_by_model, "by_category": run_by_category}
+
+
+if __name__ == "__main__":
+    import argparse
+
+    HERE = Path(__file__).resolve().parent
+    ap = argparse.ArgumentParser(description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--config", type=Path, default=HERE / "ci_recall_tables.json",
+                    help="JSON defining domains, error-type categories, and tables. "
+                         "Paths inside it are relative to the config's directory. "
+                         "Defaults to ci_recall_tables.json.")
+    args = ap.parse_args()
+    config = json.loads(args.config.read_text())
+    root = args.config.resolve().parent / config.get("results_dir", "results")
+    for table in config["tables"]:
+        kind = table["kind"]
+        if kind not in DISPATCH:
+            ap.error(f"unknown table kind {kind!r}; choices: {list(DISPATCH)}")
+        print(f"\n=== {table['title']} ===")
+        DISPATCH[kind](config, root, table)

tests/test_ci_recall.py

@@ -0,0 +1,96 @@
+"""Unit tests for the perturbation recall + bootstrap-CI helpers (ci_recall.py).
+
+Uses tiny in-memory (detected, injected) counts (no result files), so the math
+is checkable by hand: papers whose per-paper recall is a round fraction, and a
+degenerate case where every paper has the same recall so the CI collapses onto
+the point estimate.
+"""
+
+import math
+import sys
+from pathlib import Path
+
+import numpy as np
+import pytest
+
+_PERT = Path(__file__).resolve().parents[1] / "benchmarks" / "perturbation"
+if str(_PERT) not in sys.path:
+    sys.path.insert(0, str(_PERT))
+
+import ci_recall  # noqa: E402
+from ci_recall import DISPATCH, boot_ci, paper_rows  # noqa: E402
+
+
+CATEGORIES = {"surface": "Surface", "logic": "Reasoning"}
+
+
+def _per_paper():
+    """Two papers, each with surface + logic error types.
+
+    p1: surface 1/2 + logic 2/2  -> 3 detected of 4 injected.
+    p2: surface 0/2 only         -> 0 detected of 2 injected.
+    """
+    return {
+        "d/p1": {"surface": (1, 2), "logic": (2, 2)},
+        "d/p2": {"surface": (0, 2)},
+    }
+
+
+# ---- paper_rows: per-paper aggregation and category filtering ----
+
+def test_paper_rows_sums_error_types_per_paper():
+    assert paper_rows(_per_paper()) == [(3, 4), (0, 2)]
+
+
+def test_paper_rows_filters_by_category():
+    # Surface keeps both papers' surface counts.
+    assert paper_rows(_per_paper(), CATEGORIES, "Surface") == [(1, 2), (0, 2)]
+    # Reasoning (logic) exists only on p1; p2 is dropped.
+    assert paper_rows(_per_paper(), CATEGORIES, "Reasoning") == [(2, 2)]
+
+
+def test_paper_rows_drops_zero_injected_paper():
+    per_paper = {"d/p1": {"surface": (1, 2)}, "d/p3": {"surface": (0, 0)}}
+    assert paper_rows(per_paper) == [(1, 2)]
+
+
+# ---- boot_ci: pooled recall point estimate and CI ----
+
+def test_boot_ci_point_estimate_and_counts():
+    point, lo, hi, det, inj = boot_ci([(3, 4), (0, 2)])
+    assert point == 0.5 and det == 3 and inj == 6
+    assert 0.0 <= lo <= point <= hi <= 1.0
+
+
+def test_boot_ci_pins_exact_bounds_under_fixed_seed():
+    # Reseed the module generator so the resampling is reproducible, then pin the
+    # exact percentile bounds for papers with genuine between-paper variance. This
+    # checks the percentile core itself, which the inequality assertions above and
+    # the degenerate case below cannot.
+    saved = ci_recall.RNG
+    ci_recall.RNG = np.random.default_rng(0)
+    try:
+        point, lo, hi, detected, injected = boot_ci([(1, 2), (3, 4), (0, 5)])
+    finally:
+        ci_recall.RNG = saved
+    assert (detected, injected) == (4, 11)
+    assert point == pytest.approx(4 / 11)
+    assert lo == pytest.approx(0.0)
+    assert hi == pytest.approx(0.75)
+    assert lo < point < hi  # a real spread, not a collapsed interval
+
+
+def test_boot_ci_degenerate_equal_recall_collapses_ci():
+    # Every paper has recall 1/2, so every resample pools to 1/2 exactly.
+    point, lo, hi, det, inj = boot_ci([(1, 2), (3, 6)])
+    assert point == 0.5 and lo == 0.5 and hi == 0.5
+
+
+def test_boot_ci_empty_is_nan():
+    point, lo, hi, det, inj = boot_ci([])
+    assert math.isnan(point) and math.isnan(lo) and math.isnan(hi)
+    assert det == 0 and inj == 0
+
+
+def test_dispatch_covers_the_two_kinds():
+    assert set(DISPATCH) == {"by_model", "by_category"}