From 72952e87aee419ef8d7a71901c5f582d60a54682 Mon Sep 17 00:00:00 2001 From: Dang Nguyen Date: Thu, 21 May 2026 16:07:36 -0500 Subject: [PATCH] gitignore: blanket-ignore benchmarks/perturbation/configs/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the per-pattern config ignore list with a single `configs/` rule, and untracks the 5 example configs that were previously checked in. Configs/ is effectively a local experiment log — every per-domain × per-system YAML accumulates here as we iterate, and they don't survive code review or carry useful history. Files removed from tracking but kept on disk: - default.yaml, coarse_medium.yaml, coarse_short.yaml - surface_errors.yaml, surface_errors_medium.yaml Follow-up after PR #85 lands: untrack the 8 full_*_reviewer3.yaml configs it adds, since they're effectively local convention rather than something the test/CI matrix references. --- benchmarks/perturbation/.gitignore | 9 +++--- .../perturbation/configs/coarse_medium.yaml | 16 ---------- .../perturbation/configs/coarse_short.yaml | 16 ---------- benchmarks/perturbation/configs/default.yaml | 32 ------------------- .../perturbation/configs/surface_errors.yaml | 18 ----------- .../configs/surface_errors_medium.yaml | 18 ----------- 6 files changed, 5 insertions(+), 104 deletions(-) delete mode 100644 benchmarks/perturbation/configs/coarse_medium.yaml delete mode 100644 benchmarks/perturbation/configs/coarse_short.yaml delete mode 100644 benchmarks/perturbation/configs/default.yaml delete mode 100644 benchmarks/perturbation/configs/surface_errors.yaml delete mode 100644 benchmarks/perturbation/configs/surface_errors_medium.yaml diff --git a/benchmarks/perturbation/.gitignore b/benchmarks/perturbation/.gitignore index bddf55d..5cce8cb 100644 --- a/benchmarks/perturbation/.gitignore +++ b/benchmarks/perturbation/.gitignore @@ -4,12 +4,13 @@ _temp_results/ # Sample paper sources (data, not code; users supply their own) papers/ +data/ -# Run logs -reports/*.log +# Reports (regeneratable from results/) +reports/ -# Temporary / ephemeral configs -configs/_* +# Local experiment configs (we don't check any of these in). +configs/ # Python __pycache__/ diff --git a/benchmarks/perturbation/configs/coarse_medium.yaml b/benchmarks/perturbation/configs/coarse_medium.yaml deleted file mode 100644 index 5b34002..0000000 --- a/benchmarks/perturbation/configs/coarse_medium.yaml +++ /dev/null @@ -1,16 +0,0 @@ -max_papers: 5 -length: medium -error_type: surface -score_method: llm - -perturb_model: google/gemini-3-flash-preview -score_model: google/gemini-3-flash-preview - -# Same three models as surface_errors_medium.yaml (openaireview run) -coarse_models: - - google/gemini-3-flash-preview - - qwen/qwen3-235b-a22b-2507 - - z-ai/glm-4.6 - -# Shares perturb artifacts with the existing surface_errors_medium.yaml run -results_dir: benchmarks/perturbation/results_medium diff --git a/benchmarks/perturbation/configs/coarse_short.yaml b/benchmarks/perturbation/configs/coarse_short.yaml deleted file mode 100644 index 2594575..0000000 --- a/benchmarks/perturbation/configs/coarse_short.yaml +++ /dev/null @@ -1,16 +0,0 @@ -max_papers: 5 -length: short -error_type: surface -score_method: llm - -perturb_model: google/gemini-3-flash-preview -score_model: google/gemini-3-flash-preview - -# Same three models as surface_errors.yaml (openaireview run) for apples-to-apples -coarse_models: - - google/gemini-3-flash-preview - - qwen/qwen3-235b-a22b-2507 - - z-ai/glm-4.6 - -# Shares perturb artifacts with the existing surface_errors.yaml run -results_dir: benchmarks/perturbation/results_short diff --git a/benchmarks/perturbation/configs/default.yaml b/benchmarks/perturbation/configs/default.yaml deleted file mode 100644 index 9031267..0000000 --- a/benchmarks/perturbation/configs/default.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Default config for benchmarks/perturbation/run_pipeline.py -# -# Usage: -# python run_pipeline.py configs/default.yaml -# -# To run a variant, copy this file, edit, and point at the new file. -# Committed configs serve as the experiment log. - -max_papers: 2 -length: short # short (2k-7k words) | medium (7k-17k) | long (>17k) - -score_method: llm # llm | fuzzy | semantic - -perturb_model: google/gemini-3-flash-preview -score_model: google/gemini-3-flash-preview - -# Substantive-error verifier (validation test #5). Runs after the 4 structural -# tests. Strong-model oracle that drops typo-shaped / not-an-error perturbations. -verifier_model: anthropic/claude-sonnet-4-6 -verifier_reasoning: none # none | low | medium | high -verifier_max_workers: 8 -skip_verifier: false - -review_models: - - google/gemini-3-flash-preview - -review_methods: - - zero_shot - - local - - progressive - -results_dir: benchmarks/perturbation/results diff --git a/benchmarks/perturbation/configs/surface_errors.yaml b/benchmarks/perturbation/configs/surface_errors.yaml deleted file mode 100644 index 07d3ec3..0000000 --- a/benchmarks/perturbation/configs/surface_errors.yaml +++ /dev/null @@ -1,18 +0,0 @@ -max_papers: 5 -length: short -error_type: surface -score_method: llm - -perturb_model: google/gemini-3-flash-preview -score_model: google/gemini-3-flash-preview - -review_models: - - google/gemini-3-flash-preview - - z-ai/glm-4.6 - - qwen/qwen3-235b-a22b-2507 - -review_methods: - - zero_shot - - progressive - -results_dir: benchmarks/perturbation/results_short diff --git a/benchmarks/perturbation/configs/surface_errors_medium.yaml b/benchmarks/perturbation/configs/surface_errors_medium.yaml deleted file mode 100644 index 0fc44bb..0000000 --- a/benchmarks/perturbation/configs/surface_errors_medium.yaml +++ /dev/null @@ -1,18 +0,0 @@ -max_papers: 5 -length: medium -error_type: surface -score_method: llm - -perturb_model: google/gemini-3-flash-preview -score_model: google/gemini-3-flash-preview - -review_models: - - google/gemini-3-flash-preview - - z-ai/glm-4.6 - - qwen/qwen3-235b-a22b-2507 - -review_methods: - - zero_shot - - progressive - -results_dir: benchmarks/perturbation/results_medium