diff --git a/benchmarks/perturbation/.gitignore b/benchmarks/perturbation/.gitignore index 5cce8cb..175d040 100644 --- a/benchmarks/perturbation/.gitignore +++ b/benchmarks/perturbation/.gitignore @@ -20,3 +20,6 @@ __pycache__/ .ipynb_checkpoints/ *.pdf + +# Generated figures +plots/ diff --git a/benchmarks/perturbation/ci_recall.py b/benchmarks/perturbation/ci_recall.py new file mode 100644 index 0000000..d6f5cc2 --- /dev/null +++ b/benchmarks/perturbation/ci_recall.py @@ -0,0 +1,177 @@ +"""95% confidence intervals for the recall tables in the paper. + +Recall here is the fraction of injected errors that a review system detects. The +intervals come from a cluster bootstrap, where the cluster is a paper: collect every +paper's (detected, injected) error counts for a given method and model, resample those +papers with replacement many times, recompute the pooled recall (total detected divided +by total injected) on each resample, and take the 2.5th and 97.5th percentiles. +Resampling whole papers makes the interval reflect how much the estimate would move if +the paper sample changed. Pass an error category to restrict a cell to one error type. + +The paper sets, result directories, and tables to print live in a JSON config (--config, +defaults to ci_recall_tables.json), kept local because it names internal result +directories and model ids. The bootstrap math is covered by tests/test_ci_recall.py on +in-memory counts. Two table kinds: by_model (recall per model and method) and by_category +(recall split by error category for selected systems). + +Every cell draws from one seeded random generator, consumed in config order. A table's +CIs reproduce exactly only when the whole config runs unchanged, so adding or reordering +cells shifts the intervals of later ones. +""" +import json +from pathlib import Path + +import numpy as np + +B = 5000 +RNG = np.random.default_rng(42) + + +# ---- pure aggregation + bootstrap (unit-tested in tests/test_ci_recall.py) ---- + +def paper_rows(per_paper, category_of=None, category=None): + """Per-paper (detected, injected) counts for one table cell. + + `per_paper` maps a paper id to its per-error-type counts, each a + (detected, injected) pair. With `category` set, only error types whose + `category_of[error_type]` equals it are summed, restricting the cell to one + category (e.g. Surface). A paper is dropped when it contributes no matching + error type or zero injected errors. + """ + category_of = category_of or {} + rows = [] + for counts_by_type in per_paper.values(): + detected = injected = 0 + matched = False + for error_type, (n_detected, n_injected) in counts_by_type.items(): + if category and category_of.get(error_type) != category: + continue + detected += n_detected + injected += n_injected + matched = True + if matched and injected > 0: + rows.append((detected, injected)) + return rows + + +def boot_ci(rows): + """Pooled recall and its 95% cluster-bootstrap CI for one cell. + + `rows` is the per-paper (detected, injected) list from paper_rows. Returns + (point, lo, hi, detected, injected): the pooled recall + sum(detected) / sum(injected), the 2.5/97.5 bootstrap percentiles over + paper resamples, and the summed counts. All-nan with zeroed counts when + `rows` is empty. + """ + if not rows: + return (float("nan"), float("nan"), float("nan"), 0, 0) + det = np.array([r[0] for r in rows], float) + inj = np.array([r[1] for r in rows], float) + point = det.sum() / inj.sum() + n = len(rows) + idx = RNG.integers(0, n, size=(B, n)) + rec = det[idx].sum(axis=1) / inj[idx].sum(axis=1) + lo, hi = np.percentile(rec, [2.5, 97.5]) + return (point, lo, hi, int(det.sum()), int(inj.sum())) + + +# ---- reading score files off disk ---- + +def base_of(domain): + return domain.replace("full_", "").replace("_scaleup_v2", "") + + +def load_score(root, score, domain, model, etype, method, paper): + if method == "reviewer3": + p = root / f"full_{base_of(domain)}_reviewer3" / "reviewer3" / etype / "reviewer3" / paper / "score" / score / f"{paper}_score.json" + else: + p = root / domain / model / etype / method / paper / "score" / score / f"{paper}_score.json" + if not p.exists(): + return None + return json.loads(p.read_text()) + + +def load_cell(config, root, model_slug, method): + """Read each paper's per-error-type (detected, injected) counts for a cell. + + Returns {paper_id: {error_type: (detected, injected)}} over the domains in + the config, ready for paper_rows. Each (domain, paper) is its own cluster, + so the same paper id under different domains stays separate. + """ + per_paper = {} + for dom in config["domains"]: + domain = dom["dir"] + for paper in dom["papers"]: + counts = {} + for etype in dom["error_types"]: + s = load_score(root, config["score"], domain, model_slug, etype, method, paper) + if s is None: + continue + counts[etype] = (s["n_detected"], s["n_injected"]) + if counts: + per_paper[f"{domain}/{paper}"] = counts + return per_paper + + +def cell_ci(config, root, slug, method, category=None): + """(boot_ci tuple, n_papers) for one (model, method[, category]) cell.""" + rows = paper_rows(load_cell(config, root, slug, method), config.get("categories"), category) + return boot_ci(rows), len(rows) + + +# ---- table renderers ---- + +def fmt(point, lo, hi, d, i): + if i == 0: # no injected errors loaded: empty category or a misconfigured cell + return f"{'NO DATA':^30}" + return f"{point*100:5.1f}% [{lo*100:4.1f}, {hi*100:4.1f}] ({d}/{i})" + + +def run_by_model(config, root, table): + """Recall per model x method, plus any single-cell extra rows (e.g. Reviewer3).""" + methods = table["methods"] + for label, slug in table["models"].items(): + line = f"{label:22s}" + for method in methods: + ci, _ = cell_ci(config, root, slug, method) + line += " | " + fmt(*ci) + print(line) + for extra in table.get("extra_rows", []): + ci, _ = cell_ci(config, root, extra["slug"], extra["method"]) + print(f"{extra['label']:22s}| " + fmt(*ci)) + + +def run_by_category(config, root, table): + """Recall split by error category for selected (system) cells.""" + cats = table["categories"] + print(f"{'cell':24s} | " + " | ".join((c or "Overall") for c in cats)) + for cell in table["cells"]: + line = f"{cell['label']:24s}" + for c in cats: + (p, lo, hi, d, i), n = cell_ci(config, root, cell["slug"], cell["method"], c) + line += " | " + ("NO DATA" if n == 0 else f"{p*100:4.1f}[{lo*100:.1f},{hi*100:.1f}]n{n}") + print(line) + + +DISPATCH = {"by_model": run_by_model, "by_category": run_by_category} + + +if __name__ == "__main__": + import argparse + + HERE = Path(__file__).resolve().parent + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--config", type=Path, default=HERE / "ci_recall_tables.json", + help="JSON defining domains, error-type categories, and tables. " + "Paths inside it are relative to the config's directory. " + "Defaults to ci_recall_tables.json.") + args = ap.parse_args() + config = json.loads(args.config.read_text()) + root = args.config.resolve().parent / config.get("results_dir", "results") + for table in config["tables"]: + kind = table["kind"] + if kind not in DISPATCH: + ap.error(f"unknown table kind {kind!r}; choices: {list(DISPATCH)}") + print(f"\n=== {table['title']} ===") + DISPATCH[kind](config, root, table) diff --git a/tests/test_ci_recall.py b/tests/test_ci_recall.py new file mode 100644 index 0000000..7689e29 --- /dev/null +++ b/tests/test_ci_recall.py @@ -0,0 +1,96 @@ +"""Unit tests for the perturbation recall + bootstrap-CI helpers (ci_recall.py). + +Uses tiny in-memory (detected, injected) counts (no result files), so the math +is checkable by hand: papers whose per-paper recall is a round fraction, and a +degenerate case where every paper has the same recall so the CI collapses onto +the point estimate. +""" + +import math +import sys +from pathlib import Path + +import numpy as np +import pytest + +_PERT = Path(__file__).resolve().parents[1] / "benchmarks" / "perturbation" +if str(_PERT) not in sys.path: + sys.path.insert(0, str(_PERT)) + +import ci_recall # noqa: E402 +from ci_recall import DISPATCH, boot_ci, paper_rows # noqa: E402 + + +CATEGORIES = {"surface": "Surface", "logic": "Reasoning"} + + +def _per_paper(): + """Two papers, each with surface + logic error types. + + p1: surface 1/2 + logic 2/2 -> 3 detected of 4 injected. + p2: surface 0/2 only -> 0 detected of 2 injected. + """ + return { + "d/p1": {"surface": (1, 2), "logic": (2, 2)}, + "d/p2": {"surface": (0, 2)}, + } + + +# ---- paper_rows: per-paper aggregation and category filtering ---- + +def test_paper_rows_sums_error_types_per_paper(): + assert paper_rows(_per_paper()) == [(3, 4), (0, 2)] + + +def test_paper_rows_filters_by_category(): + # Surface keeps both papers' surface counts. + assert paper_rows(_per_paper(), CATEGORIES, "Surface") == [(1, 2), (0, 2)] + # Reasoning (logic) exists only on p1; p2 is dropped. + assert paper_rows(_per_paper(), CATEGORIES, "Reasoning") == [(2, 2)] + + +def test_paper_rows_drops_zero_injected_paper(): + per_paper = {"d/p1": {"surface": (1, 2)}, "d/p3": {"surface": (0, 0)}} + assert paper_rows(per_paper) == [(1, 2)] + + +# ---- boot_ci: pooled recall point estimate and CI ---- + +def test_boot_ci_point_estimate_and_counts(): + point, lo, hi, det, inj = boot_ci([(3, 4), (0, 2)]) + assert point == 0.5 and det == 3 and inj == 6 + assert 0.0 <= lo <= point <= hi <= 1.0 + + +def test_boot_ci_pins_exact_bounds_under_fixed_seed(): + # Reseed the module generator so the resampling is reproducible, then pin the + # exact percentile bounds for papers with genuine between-paper variance. This + # checks the percentile core itself, which the inequality assertions above and + # the degenerate case below cannot. + saved = ci_recall.RNG + ci_recall.RNG = np.random.default_rng(0) + try: + point, lo, hi, detected, injected = boot_ci([(1, 2), (3, 4), (0, 5)]) + finally: + ci_recall.RNG = saved + assert (detected, injected) == (4, 11) + assert point == pytest.approx(4 / 11) + assert lo == pytest.approx(0.0) + assert hi == pytest.approx(0.75) + assert lo < point < hi # a real spread, not a collapsed interval + + +def test_boot_ci_degenerate_equal_recall_collapses_ci(): + # Every paper has recall 1/2, so every resample pools to 1/2 exactly. + point, lo, hi, det, inj = boot_ci([(1, 2), (3, 6)]) + assert point == 0.5 and lo == 0.5 and hi == 0.5 + + +def test_boot_ci_empty_is_nan(): + point, lo, hi, det, inj = boot_ci([]) + assert math.isnan(point) and math.isnan(lo) and math.isnan(hi) + assert det == 0 and inj == 0 + + +def test_dispatch_covers_the_two_kinds(): + assert set(DISPATCH) == {"by_model", "by_category"}