From 120e9730d745d68dbea6d779f5545ae90f946afa Mon Sep 17 00:00:00 2001 From: MohammedPathariya Date: Wed, 22 Jul 2026 22:47:36 -0400 Subject: [PATCH] add guarded benchmark reports --- backend/benchmarks.py | 10 +- .../report.json | 294 ++++++++++++++++++ .../report.json | 294 ++++++++++++++++++ benchmark/baseline.py | 82 ++++- benchmark/digital_forge.py | 22 ++ benchmark/models.py | 32 ++ docs/BENCHMARK_GUARDED_RUN_2026_07_23.md | 48 +++ docs/STATUS.md | 31 +- frontend/app/benchmark/page.tsx | 38 +-- frontend/app/globals.css | 50 ++- .../data_level0.bin | Bin 116400 -> 116400 bytes rag/index/v1/chroma.sqlite3 | Bin 233472 -> 233472 bytes tests/test_baseline.py | 92 +++++- tests/test_benchmarks.py | 30 ++ 14 files changed, 980 insertions(+), 43 deletions(-) create mode 100644 benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/report.json create mode 100644 benchmark-results/46e08fb1861141cf82fad76ef676ce5b/report.json create mode 100644 docs/BENCHMARK_GUARDED_RUN_2026_07_23.md diff --git a/backend/benchmarks.py b/backend/benchmarks.py index 3e52eb4..2da68ba 100644 --- a/backend/benchmarks.py +++ b/backend/benchmarks.py @@ -19,4 +19,12 @@ def load_benchmark_reports(root: Path) -> tuple[BenchmarkReport, ...]: ).benchmark_version == BENCHMARK_VERSION ] - return tuple(sorted(reports, key=lambda report: report.completed_at, reverse=True)) + return tuple( + sorted( + reports, + key=lambda report: ( + not report.model.startswith("digital-forge:"), + -report.completed_at.timestamp(), + ), + ) + ) diff --git a/benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/report.json b/benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/report.json new file mode 100644 index 0000000..6f06129 --- /dev/null +++ b/benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/report.json @@ -0,0 +1,294 @@ +{ + "schema_version": "1.1.0", + "benchmark_version": "1.1.0", + "evaluator_sha256": "cc95a687dc7e8200dde2700ad87d744b3f0dbe004bd6d9285c17bb3beefc3f2b", + "run_id": "3da7bfb192c84a47aedc04a1d20be0f7", + "model": "gpt-4o-mini", + "sandbox_backend": "docker", + "started_at": "2026-07-16T22:08:53.162884Z", + "completed_at": "2026-07-16T22:10:00.087609Z", + "tasks_passed": 14, + "tasks_total": 20, + "results": [ + { + "task_id": "forge_easy_01", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": false, + "tests_passed": 3, + "tests_total": 4, + "duration_seconds": 3.2775099579885136, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_easy_01.py", + "response_id": "resp_01dee794ab8dd589016a59567677fc819490fec143ddc4fe2d", + "input_tokens": 136, + "output_tokens": 147, + "error": "hidden case failed during hidden case 3" + }, + { + "task_id": "forge_easy_02", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.302995167003246, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_easy_02.py", + "response_id": "resp_052df5c00c8d8c84016a59567d93e88195bbd63bc9f7cc11ba", + "input_tokens": 187, + "output_tokens": 124, + "error": null + }, + { + "task_id": "forge_easy_03", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.35199062500032596, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_easy_03.py", + "response_id": "resp_0defad304ea562aa016a595681aba08194b077d0ee4c92f9c1", + "input_tokens": 160, + "output_tokens": 134, + "error": null + }, + { + "task_id": "forge_easy_04", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.35644937501638196, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_easy_04.py", + "response_id": "resp_0a56148735cc227b016a595684f57c819485b65f5f0d5a1aad", + "input_tokens": 162, + "output_tokens": 127, + "error": null + }, + { + "task_id": "forge_easy_05", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.3724028750148136, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_easy_05.py", + "response_id": "resp_015ec888d0cb8a23016a595687719c81959acc5b247b07fa0e", + "input_tokens": 175, + "output_tokens": 173, + "error": null + }, + { + "task_id": "forge_easy_06", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": false, + "tests_passed": 0, + "tests_total": 4, + "duration_seconds": 0.433890040992992, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_easy_06.py", + "response_id": "resp_0a91472275072263016a59568a09f08196b143909a3b41ff9f", + "input_tokens": 127, + "output_tokens": 101, + "error": "hidden case failed during hidden case 0" + }, + { + "task_id": "forge_easy_07", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": false, + "tests_passed": 2, + "tests_total": 4, + "duration_seconds": 0.2925869589962531, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_easy_07.py", + "response_id": "resp_05d16c4d0b7a3a94016a59568cd6b48197bcd39397d63aba9d", + "input_tokens": 156, + "output_tokens": 119, + "error": "hidden case failed during hidden case 2" + }, + { + "task_id": "forge_easy_08", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.22496262498316355, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_easy_08.py", + "response_id": "resp_0e79bff60b773e73016a59568fae2c8195aa258139b5c5c6d4", + "input_tokens": 157, + "output_tokens": 98, + "error": null + }, + { + "task_id": "forge_easy_09", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.3621526249917224, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_easy_09.py", + "response_id": "resp_07e015a27f18a9dd016a595692279481948eeb266697f83464", + "input_tokens": 147, + "output_tokens": 135, + "error": null + }, + { + "task_id": "forge_easy_10", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.33703416597563773, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_easy_10.py", + "response_id": "resp_0c376e4bf2a266e5016a5956946ebc8195b282891baa080b57", + "input_tokens": 152, + "output_tokens": 149, + "error": null + }, + { + "task_id": "forge_medium_01", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.30225487498682924, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_medium_01.py", + "response_id": "resp_05c2c6e546601458016a595696a27c8194bf7527ac420d907a", + "input_tokens": 194, + "output_tokens": 237, + "error": null + }, + { + "task_id": "forge_medium_02", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": false, + "tests_passed": 1, + "tests_total": 4, + "duration_seconds": 0.32492045799153857, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_medium_02.py", + "response_id": "resp_0ea50ab90a61de9e016a595699ea10819583c0d18d94218f58", + "input_tokens": 168, + "output_tokens": 285, + "error": "hidden case failed during hidden case 1" + }, + { + "task_id": "forge_medium_03", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.35062162499525584, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_medium_03.py", + "response_id": "resp_0dcadecaf3981c52016a59569e211c819487912a4b000acebd", + "input_tokens": 158, + "output_tokens": 201, + "error": null + }, + { + "task_id": "forge_medium_04", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": false, + "tests_passed": 0, + "tests_total": 4, + "duration_seconds": 0.307560207991628, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_medium_04.py", + "response_id": "resp_07eb725e43f1972a016a5956a156a08193879597b2392dcd79", + "input_tokens": 168, + "output_tokens": 177, + "error": "hidden case failed during hidden case 0" + }, + { + "task_id": "forge_medium_05", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.3377802910108585, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_medium_05.py", + "response_id": "resp_0770e6c978b93d2d016a5956a3e1ac8197bfedcf67e31cc539", + "input_tokens": 130, + "output_tokens": 182, + "error": null + }, + { + "task_id": "forge_medium_06", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": false, + "tests_passed": 1, + "tests_total": 4, + "duration_seconds": 0.2906482909747865, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_medium_06.py", + "response_id": "resp_0b130ad2167f830a016a5956a6f664819782a3e77d1a31c7ca", + "input_tokens": 143, + "output_tokens": 258, + "error": "hidden case failed during hidden case 1" + }, + { + "task_id": "forge_medium_07", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.23271266699885018, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_medium_07.py", + "response_id": "resp_0c064b2e1547ac7a016a5956ab01808193a5442698153618ca", + "input_tokens": 160, + "output_tokens": 198, + "error": null + }, + { + "task_id": "forge_medium_08", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.2234365420008544, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_medium_08.py", + "response_id": "resp_09999fdc45ea51cd016a5956ade4bc8195a52573be4ef8457b", + "input_tokens": 161, + "output_tokens": 138, + "error": null + }, + { + "task_id": "forge_medium_09", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.88722304199473, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_medium_09.py", + "response_id": "resp_0c12e4c5b4b7431d016a5956b05e0c8190a7b3f26ba41a42c7", + "input_tokens": 143, + "output_tokens": 190, + "error": null + }, + { + "task_id": "forge_medium_10", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.7286064169893507, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_medium_10.py", + "response_id": "resp_05b994bfc13f379a016a5956b4afe88190ac4d77e1072b516e", + "input_tokens": 182, + "output_tokens": 161, + "error": null + } + ] +} \ No newline at end of file diff --git a/benchmark-results/46e08fb1861141cf82fad76ef676ce5b/report.json b/benchmark-results/46e08fb1861141cf82fad76ef676ce5b/report.json new file mode 100644 index 0000000..c8f1414 --- /dev/null +++ b/benchmark-results/46e08fb1861141cf82fad76ef676ce5b/report.json @@ -0,0 +1,294 @@ +{ + "schema_version": "1.1.0", + "benchmark_version": "1.1.0", + "evaluator_sha256": "cc95a687dc7e8200dde2700ad87d744b3f0dbe004bd6d9285c17bb3beefc3f2b", + "run_id": "46e08fb1861141cf82fad76ef676ce5b", + "model": "digital-forge:gpt-4o-mini", + "sandbox_backend": "docker", + "started_at": "2026-07-23T02:11:13.710275Z", + "completed_at": "2026-07-23T02:20:27.497262Z", + "tasks_passed": 16, + "tasks_total": 20, + "results": [ + { + "task_id": "forge_easy_01", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.26150158399832435, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_easy_01.py", + "response_id": "69d3eafa-439e-4fcd-beb5-d695e20fff66", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_easy_02", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.26355545800106484, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_easy_02.py", + "response_id": "7143886a-d87b-4b30-ae96-949321bd3434", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_easy_03", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.36002683399783564, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_easy_03.py", + "response_id": "d0541f31-a036-491d-bd64-35a79799b4e3", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_easy_04", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.29730383299829555, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_easy_04.py", + "response_id": "db14f55c-5bb3-411a-bdd0-9ccafdda4954", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_easy_05", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.2883760420008912, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_easy_05.py", + "response_id": "c9db6288-0b97-4020-a5a4-e4ea6af79aec", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_easy_06", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.25545824999790057, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_easy_06.py", + "response_id": "77a024da-f623-45d0-9d6d-72d9c56a9b51", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_easy_07", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": false, + "tests_passed": 2, + "tests_total": 4, + "duration_seconds": 0.3405088330000581, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_easy_07.py", + "response_id": "f6e5bccb-bf27-41b3-85c5-7396ffe4a4f2", + "input_tokens": null, + "output_tokens": null, + "error": "hidden case failed during hidden case 2" + }, + { + "task_id": "forge_easy_08", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.22228470900154207, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_easy_08.py", + "response_id": "0a1770ed-70cd-4f65-a55b-f737c2e61273", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_easy_09", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.2681217919998744, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_easy_09.py", + "response_id": "e071dcb0-2f21-4212-9984-3a703ef30138", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_easy_10", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.3436138329998357, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_easy_10.py", + "response_id": "f79da7f5-7daf-48ee-a2da-bda3c17bff6b", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_medium_01", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": false, + "tests_passed": 0, + "tests_total": 4, + "duration_seconds": 0.41220925000015995, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_medium_01.py", + "response_id": "c68e2146-2aa3-41a0-af2c-7fd20373709a", + "input_tokens": null, + "output_tokens": null, + "error": "hidden case failed during hidden case 0" + }, + { + "task_id": "forge_medium_02", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": false, + "tests_passed": 1, + "tests_total": 4, + "duration_seconds": 0.2623956659990654, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_medium_02.py", + "response_id": "008c315b-3ad3-46a4-9e20-6a1eb3973bd2", + "input_tokens": null, + "output_tokens": null, + "error": "hidden case failed during hidden case 1" + }, + { + "task_id": "forge_medium_03", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.4514182079983584, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_medium_03.py", + "response_id": "f81f88a2-af13-4366-bf69-36936e0be47d", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_medium_04", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.24314154199964833, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_medium_04.py", + "response_id": "fd746bd4-7ef5-47c2-995d-a5ccdc0a819c", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_medium_05", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.25285216600241256, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_medium_05.py", + "response_id": "304af63f-2425-40dc-801f-f2bb14ea1b59", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_medium_06", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": false, + "tests_passed": 0, + "tests_total": 4, + "duration_seconds": 0.24811241600036738, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_medium_06.py", + "response_id": "8d06c68d-52d5-4e46-bc21-d4f020f96b26", + "input_tokens": null, + "output_tokens": null, + "error": "hidden case failed during hidden case 0" + }, + { + "task_id": "forge_medium_07", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.22136095799942268, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_medium_07.py", + "response_id": "0db6d8bc-ea14-44cc-8b10-824bd6272a77", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_medium_08", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.30464370899790083, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_medium_08.py", + "response_id": "9709893a-479d-440a-a2cc-c662a11878c4", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_medium_09", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.3257320830016397, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_medium_09.py", + "response_id": "f019256c-7f89-4f5e-b694-7691e37aa57f", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_medium_10", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.3562291250018461, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_medium_10.py", + "response_id": "43b00afa-9848-4b05-9a5a-49560ec9f052", + "input_tokens": null, + "output_tokens": null, + "error": null + } + ] +} \ No newline at end of file diff --git a/benchmark/baseline.py b/benchmark/baseline.py index 62ecab1..8646c4c 100644 --- a/benchmark/baseline.py +++ b/benchmark/baseline.py @@ -20,6 +20,7 @@ from .evaluator import evaluate_candidate from .hidden_cases import evaluator_sha256 from .models import ( + BenchmarkInterruptedReport, BenchmarkReport, BenchmarkTask, GeneratedSolution, @@ -73,20 +74,72 @@ def __init__( model: str, output_root: Path, sandbox_runner: SandboxRunner | None = None, + max_consecutive_failures: int | None = None, + finish_remaining_threshold: int = 3, ): self.generator = generator self.model = model self.output_root = output_root self.sandbox_runner = sandbox_runner or DockerSandboxRunner() + self.max_consecutive_failures = max_consecutive_failures + self.finish_remaining_threshold = finish_remaining_threshold - def run(self, tasks: Sequence[BenchmarkTask] | None = None) -> BenchmarkReport: + def run( + self, tasks: Sequence[BenchmarkTask] | None = None + ) -> BenchmarkReport | BenchmarkInterruptedReport: selected = tuple(tasks or load_tasks()) run_id = uuid4().hex run_directory = self.output_root / run_id candidates_directory = run_directory / "candidates" + checkpoints_directory = run_directory / "checkpoints" candidates_directory.mkdir(parents=True, exist_ok=False) + checkpoints_directory.mkdir(parents=True, exist_ok=False) started_at = utc_now() - results = tuple(self._run_task(task, candidates_directory) for task in selected) + results: list[TaskResult] = [] + consecutive_failures = 0 + stop_reason: str | None = None + + for index, task in enumerate(selected): + result = self._run_task(task, candidates_directory) + results.append(result) + result.write(checkpoints_directory / f"{task.id}.json") + + consecutive_failures = 0 if result.passed else consecutive_failures + 1 + remaining = len(selected) - index - 1 + if ( + self.max_consecutive_failures is not None + and consecutive_failures >= self.max_consecutive_failures + and remaining > self.finish_remaining_threshold + ): + stop_reason = ( + f"stopped after {consecutive_failures} consecutive failures " + f"with {remaining} tasks remaining" + ) + break + + if stop_reason is not None: + interrupted = BenchmarkInterruptedReport( + benchmark_version=BENCHMARK_VERSION, + evaluator_sha256=evaluator_sha256(), + run_id=run_id, + model=self.model, + sandbox_backend=self.sandbox_runner.name, + started_at=started_at, + interrupted_at=utc_now(), + status="stopped_by_guardrail", + stop_reason=stop_reason, + intended_tasks=len(selected), + completed_tasks=len(results), + tasks_passed=sum(result.passed for result in results), + tasks_total=len(results), + max_consecutive_failures=self.max_consecutive_failures, + finish_remaining_threshold=self.finish_remaining_threshold, + results=tuple(results), + ) + interrupted.write(run_directory / "interrupted.json") + return interrupted + + result_tuple = tuple(results) report = BenchmarkReport( benchmark_version=BENCHMARK_VERSION, evaluator_sha256=evaluator_sha256(), @@ -95,9 +148,9 @@ def run(self, tasks: Sequence[BenchmarkTask] | None = None) -> BenchmarkReport: sandbox_backend=self.sandbox_runner.name, started_at=started_at, completed_at=utc_now(), - tasks_passed=sum(result.passed for result in results), - tasks_total=len(results), - results=results, + tasks_passed=sum(result.passed for result in result_tuple), + tasks_total=len(result_tuple), + results=result_tuple, ) report.write(run_directory / "report.json") return report @@ -164,6 +217,18 @@ def main(argv: Sequence[str] | None = None) -> None: default="digital-forge-sandbox", help="Modal app used when --sandbox=modal", ) + parser.add_argument( + "--max-consecutive-failures", + type=int, + default=None, + help="Stop early after this many consecutive failures (default: disabled)", + ) + parser.add_argument( + "--finish-remaining-threshold", + type=int, + default=3, + help="Ignore the failure guard when this many or fewer tasks remain", + ) args = parser.parse_args(argv) tasks = [get_task(task_id) for task_id in args.task_ids] if args.task_ids else None sandbox_runner: SandboxRunner = ( @@ -172,7 +237,12 @@ def main(argv: Sequence[str] | None = None) -> None: else DockerSandboxRunner() ) report = ZeroShotBaselineRunner( - OpenAISolutionGenerator(), args.model, args.output, sandbox_runner + OpenAISolutionGenerator(), + args.model, + args.output, + sandbox_runner, + max_consecutive_failures=args.max_consecutive_failures, + finish_remaining_threshold=args.finish_remaining_threshold, ).run(tasks) print(report.model_dump_json(indent=2)) diff --git a/benchmark/digital_forge.py b/benchmark/digital_forge.py index 203873f..fd3102c 100644 --- a/benchmark/digital_forge.py +++ b/benchmark/digital_forge.py @@ -1,6 +1,7 @@ """Digital Forge adapter for the algorithm benchmark.""" import argparse +import os from collections.abc import Sequence from pathlib import Path @@ -65,8 +66,27 @@ def main(argv: Sequence[str] | None = None) -> None: default="digital-forge-sandbox", help="Modal app used when --sandbox=modal", ) + parser.add_argument( + "--max-consecutive-failures", + type=int, + default=None, + help="Stop early after this many consecutive failures (default: disabled)", + ) + parser.add_argument( + "--finish-remaining-threshold", + type=int, + default=3, + help="Ignore the failure guard when this many or fewer tasks remain", + ) args = parser.parse_args(argv) + runtime_directory = (args.output / ".runtime").resolve() + runtime_directory.mkdir(parents=True, exist_ok=True) + crewai_storage_directory = runtime_directory / "crewai" + crewai_storage_directory.mkdir(parents=True, exist_ok=True) + os.environ.setdefault("CREWAI_STORAGE_DIR", str(crewai_storage_directory)) + os.environ.setdefault("OTEL_SDK_DISABLED", "true") + base_settings = Settings() model_name = args.model or base_settings.openai_model_name settings = base_settings.model_copy( @@ -88,6 +108,8 @@ def main(argv: Sequence[str] | None = None) -> None: label, args.output, sandbox_runner, + max_consecutive_failures=args.max_consecutive_failures, + finish_remaining_threshold=args.finish_remaining_threshold, ).run(tasks) print(report.model_dump_json(indent=2)) diff --git a/benchmark/models.py b/benchmark/models.py index 6f20029..c564347 100644 --- a/benchmark/models.py +++ b/benchmark/models.py @@ -60,6 +60,10 @@ class TaskResult(BaseModel): output_tokens: int | None = None error: str | None = None + def write(self, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(self.model_dump_json(indent=2), encoding="utf-8") + class BenchmarkReport(BaseModel): model_config = ConfigDict(frozen=True) @@ -81,5 +85,33 @@ def write(self, path: Path) -> None: path.write_text(self.model_dump_json(indent=2), encoding="utf-8") +class BenchmarkInterruptedReport(BaseModel): + model_config = ConfigDict(frozen=True) + + schema_version: str = "1.1.0" + artifact_type: str = "interrupted_benchmark_run" + run_id: str + benchmark_version: str + evaluator_sha256: str + model: str + sandbox_backend: str + started_at: datetime + interrupted_at: datetime + status: str + stop_reason: str + intended_tasks: int = Field(ge=0) + completed_tasks: int = Field(ge=0) + tasks_passed: int = Field(ge=0) + tasks_total: int = Field(ge=0) + aggregate_score_available: bool = False + max_consecutive_failures: int | None = Field(default=None, ge=1) + finish_remaining_threshold: int = Field(ge=0) + results: tuple[TaskResult, ...] + + def write(self, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(self.model_dump_json(indent=2), encoding="utf-8") + + def utc_now() -> datetime: return datetime.now(timezone.utc) diff --git a/docs/BENCHMARK_GUARDED_RUN_2026_07_23.md b/docs/BENCHMARK_GUARDED_RUN_2026_07_23.md new file mode 100644 index 0000000..401ff31 --- /dev/null +++ b/docs/BENCHMARK_GUARDED_RUN_2026_07_23.md @@ -0,0 +1,48 @@ +# Guarded Digital Forge Benchmark Run, 2026-07-23 + +## Summary + +- Benchmark version: `1.1.0` +- Digital Forge model label: `digital-forge:gpt-4o-mini` +- Digital Forge run ID: `46e08fb1861141cf82fad76ef676ce5b` +- Digital Forge result: `16/20` +- Easy tasks: `9/10` +- Medium tasks: `7/10` +- Started: `2026-07-23T02:11:13.710275Z` +- Completed: `2026-07-23T02:20:27.497262Z` +- Report: `benchmark-results/46e08fb1861141cf82fad76ef676ce5b/report.json` + +## Baseline Comparison + +- Zero-shot model: `gpt-4o-mini` +- Zero-shot run ID: `3da7bfb192c84a47aedc04a1d20be0f7` +- Zero-shot result: `14/20` +- Zero-shot easy tasks: `7/10` +- Zero-shot medium tasks: `7/10` +- Zero-shot report: `benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/report.json` +- Net measured lift: `+2` tasks over zero-shot `gpt-4o-mini` on benchmark v1.1.0. + +The zero-shot `gpt-4` v1.1.0 run scored `15/20` and is archived at +`benchmark-results/ce0abe92172b45508794eb5e25928f70/report.json.archived`. + +## Guardrails + +- Per-task checkpoints were written to `benchmark-results/46e08fb1861141cf82fad76ef676ce5b/checkpoints/`. +- The run used `--max-consecutive-failures 3`. +- The run used `--finish-remaining-threshold 3`. +- The guardrail did not interrupt the run because no early three-failure streak occurred. + +## Failed Tasks + +| Task | Tests | Error | +| --- | ---: | --- | +| `forge_easy_07` | `2/4` | `hidden case failed during hidden case 2` | +| `forge_medium_01` | `0/4` | `hidden case failed during hidden case 0` | +| `forge_medium_02` | `1/4` | `hidden case failed during hidden case 1` | +| `forge_medium_06` | `0/4` | `hidden case failed during hidden case 0` | + +## Notes + +- This run validates the active algorithm benchmark path, not the separate RAG evaluation suite. +- Argus repair behavior was exercised during the run, including generated-test import repair paths. +- Per-agent token and cost telemetry is still not captured by the benchmark runner. diff --git a/docs/STATUS.md b/docs/STATUS.md index bf3ac37..878ab9e 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -113,6 +113,9 @@ accepted decisions. Day 7 deployment work has not started. - Changed test-owned semantic repair to discard the invalid generated suite before asking Argus for a fresh suite. This prevents incorrect expected values from anchoring subsequent repairs while preserving the original request and typed testing plan as the source of truth. +- Added benchmark checkpointing and guarded early-stop support. Each completed task now writes a + task-level checkpoint before the aggregate report is finalized, and benchmark CLI runs can stop + after a configured consecutive-failure streak unless the suite is already near completion. ## Verification performed @@ -121,6 +124,15 @@ accepted decisions. Day 7 deployment work has not started. on attempt two. Normalization run `4a93f7cd` reported success on attempt three, but its final code violated the original display-name fallback and active-string normalization rules, so it must not be counted as a successful run. Further paid prompts were stopped. +- Paid guarded benchmark verification on 2026-07-23 completed the active v1.1.0 Digital Forge + `gpt-4o-mini` run at 16/20 overall, with 9/10 easy and 7/10 medium tasks passing. The measured + report is `benchmark-results/46e08fb1861141cf82fad76ef676ce5b/report.json`, with checkpoints + under `benchmark-results/46e08fb1861141cf82fad76ef676ce5b/checkpoints/`. Failed tasks were + `forge_easy_07`, `forge_medium_01`, `forge_medium_02`, and `forge_medium_06`. +- Paid zero-shot `gpt-4` verification on 2026-07-23 completed a v1.1.0 comparison run + at 15/20 overall, with 8/10 easy and 7/10 medium tasks passing. That report is archived at + `benchmark-results/ce0abe92172b45508794eb5e25928f70/report.json.archived` while the dashboard + keeps the same-model `gpt-4o-mini` comparison active. - After the feature-flag reliability fix, `.venv/bin/pytest -q` passed with 86 tests and five environment-dependent tests skipped. Ruff checks and formatting, mypy, frontend lint and type-check, the production frontend build, and a local browser @@ -197,23 +209,22 @@ accepted decisions. Day 7 deployment work has not started. preserved artifacts and findings. - The active v1.1.0 zero-shot `gpt-4o-mini` baseline has been measured at 14/20 overall, with 7/10 easy and 7/10 medium tasks passing. The valid report is - `benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/report.json`. Two invalid - zero-token pre-runs were quarantined by renaming their `report.json` files so - the benchmark dashboard only loads the valid v1.1.0 result. + `benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/report.json`. Two invalid zero-token + pre-runs were quarantined by renaming their `report.json` files so + the benchmark dashboard only loads measured v1.1.0 results. - The first active v1.1.0 Digital Forge `gpt-4o-mini` run was stopped during `forge_easy_07` at the user's request. No aggregate score is available because the runner writes `report.json` only after all selected tasks finish. Six persisted candidates were re-evaluated locally with Docker; only `forge_easy_05` passed. The interrupted artifact is `benchmark-results/8afe20c0708f45a693c903445e75f91b/interrupted.json`. -- The structural causes found in that interrupted run are now fixed locally: stale CrewAI +- The structural causes found in that interrupted run are now fixed and remeasured: stale CrewAI tool caching is disabled, the original request is preserved through every code-producing - stage, and invalid function/import/syntax artifacts fail deterministic preflight checks. - The three semantic failures still require a small paid pilot to measure whether direct - request propagation improves live model behavior. -- The benchmark runner does not checkpoint task results or capture CrewAI token - usage and cost. Add resumable per-task reports, spending limits, and usage - telemetry before another paid Digital Forge benchmark. + stage, invalid function/import/syntax artifacts fail deterministic preflight checks, and the + guarded 2026-07-23 run completed at 16/20 on v1.1.0. +- The benchmark runner now checkpoints task results, but it still does not capture CrewAI token + usage and cost. Add spending limits, resumable execution, and usage telemetry before another + expensive benchmark run. - The local Digital Forge default model has been changed from CrewAI's effective `gpt-4` default to explicit `gpt-4o-mini` via `OPENAI_MODEL_NAME` to reduce routine localhost and pilot-run cost. Use an explicit model override only for diff --git a/frontend/app/benchmark/page.tsx b/frontend/app/benchmark/page.tsx index f4cb080..1951a58 100644 --- a/frontend/app/benchmark/page.tsx +++ b/frontend/app/benchmark/page.tsx @@ -25,7 +25,9 @@ export default function BenchmarkPage() { getBenchmarks(controller.signal) .then((data) => { setReports(data); - if (data[0]) setSelectedRun(data[0].run_id); + const primaryReport = + data.find((item) => item.model.startsWith("digital-forge:")) ?? data[0]; + if (primaryReport) setSelectedRun(primaryReport.run_id); }) .catch((reason) => { if (!controller.signal.aborted) { @@ -63,16 +65,25 @@ export default function BenchmarkPage() {

{reports.length > 1 && ( - + + )} @@ -161,10 +172,6 @@ function BenchmarkResults({ }) { const easy = report.results.filter((result) => result.difficulty === "easy"); const medium = report.results.filter((result) => result.difficulty === "medium"); - const averageDuration = report.results.length - ? report.results.reduce((total, result) => total + result.duration_seconds, 0) / - report.results.length - : 0; return ( <> @@ -191,11 +198,6 @@ function BenchmarkResults({ {percent(medium.filter((item) => item.passed).length, medium.length)}

{medium.filter((item) => item.passed).length} of {medium.length} tasks

-
- Average execution - {averageDuration.toFixed(2)}s -

Sandbox duration per task

-
diff --git a/frontend/app/globals.css b/frontend/app/globals.css index 55f4bd1..4371a60 100644 --- a/frontend/app/globals.css +++ b/frontend/app/globals.css @@ -1173,7 +1173,8 @@ button:disabled { .report-picker { display: grid; - gap: 7px; + min-width: min(340px, 100%); + gap: 8px; color: var(--text-muted); font-size: 0.68rem; font-weight: 700; @@ -1181,16 +1182,51 @@ button:disabled { text-transform: uppercase; } -.report-picker select { - min-width: 220px; - height: 42px; - padding: 0 12px; +.report-picker > span { + display: block; +} + +.report-options { + display: grid; + gap: 8px; +} + +.report-options button { + display: grid; + gap: 3px; + min-height: 54px; + padding: 10px 12px; border: 1px solid var(--border); border-radius: 8px; background: var(--surface); color: var(--text); - font-size: 0.82rem; + text-align: left; text-transform: none; + cursor: pointer; +} + +.report-options button:hover, +.report-options button.active { + border-color: var(--gold); + background: #fbfaf7; +} + +.report-options button.active { + box-shadow: inset 0 0 0 1px var(--gold); +} + +.report-options strong { + min-width: 0; + font-size: 0.83rem; + font-weight: 650; + overflow-wrap: anywhere; +} + +.report-options button span { + color: var(--text-muted); + font-family: var(--mono); + font-size: 0.67rem; + letter-spacing: 0; } .benchmark-loading { @@ -1295,7 +1331,7 @@ button:disabled { .metric-grid { display: grid; - grid-template-columns: repeat(4, 1fr); + grid-template-columns: repeat(3, 1fr); gap: 10px; } diff --git a/rag/index/v1/4bcc6f7c-634d-49b1-8be1-0c122d68b9de/data_level0.bin b/rag/index/v1/4bcc6f7c-634d-49b1-8be1-0c122d68b9de/data_level0.bin index a63e1e97d32a28646658a90e4ed56d112018b67b..cb2c1fc3a95468ba38c813c2761301d3affc62e1 100644 GIT binary patch literal 116400 zcmeI4dEnK=wa2fh$RsA$_b+6UDzS^oqj9RtsQH#}A-{VQXC-E_!`^!&mZ-2hm<3E${e9xRY^O;F* z?$4{03jW!M+FFhHwa5s+wFKmR$oXiBfSiw}=*aye=R?j%Qv~FEG(|_AA96nAd^ANs z&PP*p@OcRvP#N8fa*8;kKbPX?B=}bN@Lcj<;ggf#Tj9fV$$N!QPJ(ZR56>m<6+Sr$ zz7;+^m%LZ_7l~2BIyF1=*HzY8d8-VgSqq~se z7bw^Z;lpAnI)zVxg1rzvES92E_!KDEi)qpe9>MwKm9UtWgHpOfHQ z;lp#udxcLm<6+Sr$z7;+^m%LZ_;0LlIK^W_jgUPEyYE#WB-i%v*GKj^WPtxTJn4w zRdW6gC}^%9-wo^h`)1!d=fA$>{;PNYt}1!IsW*R8a{nzY@#~$xrN@8J;Qedq^|>YM zd#}WA>H1$S`TXeB!u3!4e|~>7djFp(dA=Q2vj6qgcTUOtW%(C2xW2Oa%lu19K7UWH zbNyxWi=V$>tl#@h)ZqO&x#azA%@;4g^#3Qx3+j|Dtls^-v*difo=?}HzklAAZ%>2r zm+@&=%0Dk~{mxI>{9kwf#g_j5SLgSmZ(5t9$>|4k+Hm*sD?e_VtAKD4aP{DVr~zrOhX7a9NkM+fXo$@yqB-?IEeO4fI4 z$@A;M62EMI@$(zFG0)$6`JPvDKI+YX$JhM+sCRw!jvrX^{OVk?KffvAJG+Iy|J_rv zzFq44e4Jb7_)Cl1vbe}^*sQI0f9oB8*4O;={jQSpSGNB5O3ruL_(uB|7ssNw_@|za zUzG6qd!77$UXs7jd>ftrq>}YrSMvR(XUX_OOU9S2Z<9L5KU(s9Et|hy|G)-6pB`)Q z`{xlQpO1^{yuWTKIY0ICzpv!)j~_L7f9kEj-tk{NjQQuY_&EXLlat_E;lp#udxcL< zf^UTn&n52_J~;`#Z7Dwayb=ArC4V99YLuDB8Qp~xzd*rW2p<+p(J6ci6zqlYVX+jQ z!lyvNUI-r+OVKHO3KZ;x@L{nOox-O;!CnX-7E947daXZYz;&}wVS z|2t^62l=Ecq}@Nj_igf@f!5ui;}=NxqI|2M6VfSZ)f>K_z_08GtzU+ANxN@TzTxEe z!}_A#DgPUke+abuSLkRYbi5kcChfloo$f*T)<9PVLt6upUyXoP9e$GW3cLOoZ9C{_ zOK5u#;wv4=-;MZphW3U*$2&tidqC@f(C!dudpBr%IJ8e%lU76FhcrGK2_20@e7NC7 zI6r$r2d$y~5#+alb}3$M0G*KU?}qh;m+34|Qh^`d4DGj~e6NC+-ii3U1zjBg?cO*a zh#hj#j6e5F6?QT?Em1K;iso$du4kninH`G&v`_ks3?LC54&_>}?BHfc2i z+TRzxcCJL7q^*6RE2KTr>CK4ucft5z474)=@fPXIcxYEsOFaCGpuL09rk7Al@d@3Z z-nF3g>1b$c zHngJn^l9p!gVuQltv-|bSE5z>Uyu3{Xn!1Z`Xa?00G+IYcE>|IuR@0tq4m4a!J(*M ze+XSY1UgJ9{xE3&Q)tz0Q8s_|@6ZnUm5-sl?;zfxd@1D*SHt%w!H+jsoaIs9g;uRe zw?Y0&Tj;P8v}zA+9fA006X=BUg&RUwDWAI$w0jiR8@*5UO~!ba^2el;PKXaEzxOt+ zZwlfoA3$plI_N_4A4&PPf>uXEN83RAkKUNg=bnqWfPDWP=(Gdc${vV!yHmdN(duo`+LYfr3Hife zl>a#BU?_C-J?ND3DXL%p1^Z?F2G4Vs-VdCU5wA4OGY#6lh~_7q%p<)3+~Xo?Nbe6_xfUF z(9yln>eAb@`t&mLzk_^M2pv#-`~Y<2ER0vAz1h(59q{#;&;j{A#XFQgA??i%vF~#c z?_B^Lk#^=mTR&Ke`W8S}eh3{egx2RmTZ^I7-SPSAbffo$(|?t{4_e)!)hx8>4Tx6* z5O3WA?JDTXt(1@6hvM6z!_zT7z6sho?YgYK=w@iO;ErrQcPZ&Zz|*|}eA3C#m8Hd$v^O7GZAbAxr}+FlTn3$<3f}JJ(A5gP zk6a1u(EFQvHO225wO#u@trtK?tr2fmp;fdvTW{q$ia+tTOzZ2RD`!F5i=Y$Qzl3~0 z-_M)iyXRp)y<4Co(#dVm>U{YA66h-F%2Mdy0{D6vbbKMS{Y&WZN6_J&&<<&RH?;o~ zioX}y{xP)oD`@wp(AE2()z6@l`=KN9{pHZsJow=X=zz4^;DKy^!yfZ+ep*4RZST)? zWh2r%K{Fco6H|6Mpq7(|7~9rb9Y-4772t5n}d(2DYTt0~_? z`1)_q-XiFPw7QBiGGl^amN7je}?{WQ|RCmXw?b2`giDbD(bU8g?6V= zz2vJ7h_8GM|MM+h=O6Wb#OrpCW%I6MG9sOc_-aSQ`!k^9_WXEF@os0{SO4}zHgDLK z_tRUztGX5Za60DmH-WaE!n&%@QM=~8?0FkK#IM&}lA5_9i+6hR^9P;qeAncAl-K$- zpWmf^=NaBlo~HgD{Ct}HusiRkH-lHwi}&5z;d?vqe!K*Jx+CwKe8Q!Ow|7B&8bYg` z`SJ0c@Pit@x)**#`ONwB{qUW>e12;=d|k`m`?uM7O8fBR)oA#Y8ov82_|?Jucs&-r zTf>jX!FPu6&f$@s+oAv7*ss8u*{Aw5Y=?A&--S4v=H&uVzlWDw2J!8_;3Pzdl6dGch04GZ{YplhgAP^-Z%4^I=t#n zSl{Vr)A?xa|9(E}H{+dG(C^PhoatMi!q@w*V#g;lDgU?Nd$jK0S;||%`;)HmzGI!X zFxy{m{`#%go`!itA8l|k^0+7Q^_uJR=dR4w6`X;1{}POkPJvFxN9gzNCfy(!-<&@SaO%Mqj z@V~|T;Z+zPkK=v+YO3ekyq_+B?;ObcRu#Uk;hTKQw7wbns#7ta-4FX19|Nrx(tJnr z^_cnNWr$CX=i|d)!VjjwPj7%$G@qHzS`Ocx&gb*{quy`^@0|Jr?ot3;)gTzeVRKT*>lE;@gv~Je}=CqzWNFu?^Aq~@V+||`K-5jKcVr-yS%UVqkp4de*YK9|0RDq{NNhik57dkRe3+12|vA#_w|`Hei83mdw0y% zAKb+IY9IL4Ej9en@ckvcZ?3mY$75-~??yg#F!G0Y^7+F<;X4#>=6Ci(KCi}jGd^s; zX~sj}kNH9xAO4E8i}`{(_)T)R4?T><2_n`v>fr#Xw+X>%FmaMfp6F1 zzd!t7uTI&zT&r#NecQT>&!?x-d3*r(xmlk-7xh=hBA@A-I-&WM%g4LlMt#viydROT zCs6%+S=lQE`0yyoHw}I`1=>52k2mv|Y2_oI zzs;7}y368C-yctWZH!mnN4}&dw0|mer3atiKA6VS{K-a$bGPE-&G}8OX3~5+))-Iq z>YZqw>4*<{L#vIcZCzu$IiaEbN4nzt^x?;+kHS~|c;84@2_Hkerw|`a0589P^Twx7 zV0AP>kPuudo8DFnXCJ~SR{Cqz7=>Xn$ z56Ae*VEF1|T7O??JrRDmD?h$M`GbD&{SPr-?OY@ND=s)V0`vQO_hg^1@pq{HQM_+X z=S@a@JerSproi{cz)xv@o96d_+&!zqao}5deIxnxR=-R0jesAoCceY@cy&4Q$J_G0 zbtU{@9ACffVLt2Iyzd_g-#q}n+5+tOgH@lHE8 zZ+^2Lbu`U)2=eKL;G-u%SIJj3)@L0kS{E{NLq$?^vuyPloTEM(;T@(WgrEh;xSU z@%l{U^GCu@DStAMkN0NNdIs^nI~Tt9_zvuTtFz%(p62~<4*CDV`_^7NXL;jic|RrJ zeUA6Ny)iy{p7+C_@0jKHR@U(MzydS;>KYEP!(-T`~ zdEF;?-(0_`!>hs5eH!Ea1^4dtWfh03*&pi0k+=&2g>=lAQAHT-?Czd=m%|7=0_ z{XRLBe?EqPq37X@8qeST`e!_Jjq&Ob_{qC9>M_=rP<{5Nh}VZ>eDz~~ytUbG+5CPh z{yg|wz}I~Tu;)Q_f$t3DeZMRG;G4W3c7q?+@QvkF{2o-#5I&#k3EvyW`>_o_8P5Cm zIP9+)%KPC#@a^4s-#!?Ay3r7pS7ki>N?YDf$k*+7KbnB?L2KUEG{4`0_p66syxWoY zgG1pbop|5-4t#5K-ggdzue$KQJ(0$D<^9Se__`bKM~Bn+?!2$QOXGX;zI6or$macc zGW?_$@7o@He|z57gyP-h`{5Zh zeq-KO`=Q_8g!jXV=y%%H@TZ_ZXwUoVX!N@sc|V+ne%*=p)#>Q>H>=^Fjeci~8vYC# zzhw=7^Sv^zVOQP{yTRApYWQ8@Tit8;z2GN3d0+LT@!Ro!H~_xWi}&3%+h+9#z2U2E z@Vs|xoG0`1B{_xepFS9$KI>%pqLKW0HOGf@=>B?SANG8vvnc-)@Z*c{{*c!Ad=7sE zKisv(dgmgazZ-wQsI%d#XZd=n!RV~c2hutN9MxmitByy!uEE3U zNOf&^Aj?aghu{19t$5#@Pxsh5+pqK+e7rvt{qYF$_vxO+tM<_P&qrtP%ki81e!D-l zGQUb3%=y2nw)-7~^RNrr*r9rQLtE!SSH1!5jDoKAfKDf4z4mg{{Z;2T9(|)#s=-Ykz;?@6X4Fe?We94DYLZaQ{Td!uLC%jrQi_qrK3d?8o~S zjq^@~AAh)cwtrD|Lbg6N@03hi*Fx*dvA@9sXwCH{_n}{Xggj|?@ON+J<12Jt(kBkj z=1Yeo&Rbj~-ki|T>W-D!e}@_V5%~g8p&mtBrS+Knx4Ptf6*4%t4K^PAc_$It55&zypm#;bey`t@B@ z=Z({|cr#y_wyK!VUV`!dWzf#`d_KK^>bo0$_ygRp{vtj;q5C;p!TV+PnSS!(%xsOoyvxW#gm6XE%$NbsFNl8EE}UXwxH_H{Pr#d2voQuX+h>x&UpJ`px(j`f|e>`O_>zCy>{qP#hXU%S2zFvNK-_M^{8()Kc`4l|#6?}Y5_n*HR;?=c?cUGY_rvH+( z+h3Q><99%->G`kQL7M{<^~W^M^xd~lN8Ov&82>!g)sgqjd}TVFgFJCRv}!u(ozeQe z(c0Ui4QT#oGw8DXWq$e+^5}`zXX{RnKPG#huN(kv(eoxA4Q=N0$Pd2_Kc@P;VSIfN zosV$W8?*T$dVfgHW24yZwLCGc_e-R+8(VLU#4Tq=f8g|c)x#3*7~nOo8yCb(I3CZ`!V_Ur@SB1_r>7R z+p~Gh{ONXBNBB7JyPeUmf6x2gW9au*^1e-VTYu#Jl*T2ic;A|Z=YmtiH`n71K%Vpi zv?0|WyvgVHslMbh-nX_ufBav(Z_aP(^wDJ*cXvG6*bOu790=`Ge)}!tSF~Tc7k|FI zZP1^-!|%5_|5w%Vyt}gcqKnX)<1OkBui}0E2CeH--mhMOe(P++McZOM!896o6m)Vj zv^l@2_1Is5=YITr?iln}$C6LyT{knoIiF}DAOwVf5D)@FKnMr{As_^VfDjM@LO=)z z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^VfDjM@LO=)z z0U;m+gn$qb0zyCt2mv7=1cZPP5CTF#2nYcoAOwVf5D)@FKnMr{As_^Vz}J<){{Sf+ BE;#@I literal 116400 zcmeHwd%PXf+5hB1E{KU+l!Or^F1ZZ3G$di_TB?kwAm}^MXenx{MXQO7s6^^EMUx<6 z$F&l4A(SYcx-@9Xw3H}`5s_4@Ohkg#<#pEgS@XVU<=I&$_V0X-{@#^;zVkfa=eEAH zW@ewg&pvS+{Od^A9KPK_**yRAcI)QHy|Zrq&|alR)PG#Rb?2MrGoD|={x#KC(>V@* z{aIcqVD(W-@M!hnbx~fck5Yn1s}HY>@>+eA5Z6q4(dxtNqP$igr38;w zA6^&bwfZO}c(nTPx+t&JM=8Oh)rZ$bd96N52_CIJye`UX^-)UjX!YTBQC_Q$Qi4aT z53h^zT78rfJX(EtU6j}Aqmci`zyjCBj1dmo9UKi!H`Y0uMwEFP6D6iE=DZ!)F zhu1}Utv*T#9<4sSF3M~5QA+S=_2G3@>+eA5Z6q4(dxtNqP$igr38;wA6^&bwfZO}c(nTPx+t&JM=8Oh z)rZ$bd96N52_CIJye`UX^-)UjX!YTBQC_Q$Qi4aT53h^zT78rfJX(EtU6j}Aqmci`zyjCBj1dmo9UKi!H`Y0uMwEFP6D6iE=DZ%6C)koz;e)HddZ2H5K?f+LR75Bef z%?>T!Opdh-tIt}JG;>&en#r-2Vf9%{l4g#!(ns~p1~$Kc4*qO9d;k2OQ>1kIA5Zfm z%Qx3Ux2!(8BF$~9PjfwV>;I}gYTZSD+@IAS{NGFyNX2RH%CvkX1$$uiVP&bDR-Z}= z_Q2}H%2GM4K9v;gL0jpg{sWX`!s@c=`=cAubgnjKTg|7&u>F-1JX(EtU6j}Aqmci`zyjCBj1dmo9UKi!H`Y0uMwEFP6D6iE=DZ!)Fhu1}Utv*T#9<4sSF3M~5QA+S= z_2G3YX3R&P4f*g=wr%nZGKgQ>pwyLx=S7XCL7LwYp<`-`nPvd0`dE| z(fZ#u?B5SISa<&)G`PPmH{_qCZWndVZ=>sP>-woAAzWy`SuLt~pdVWS5?k{b< zfBH2z-!6vh`*{OzTK^v!ynnq^-S(;jbwiS9>fhG<;{WjHPh0EP+Wj|g@bzD7&)3%V ze^gyR)fr&y@e4&X1_nY#n_+Z1DMR)VIOcH-X{jo4~ODjq@4aKk3uT z^-c3_*5K>&-VI*gYpQKm9aKBA>A*C9Yx9paxW4WUzP>xeus=r|o=+xyo^9~_o9Z87 znBSD&*8G@iH}d=Q<5t$Mwb$Rbjn7Y4!}FnL;9FatK8Du^OAY(OG~Wje>VHFn_b1bQ zCjQX|pTB=MtpC{Rva1d%#hVUH^I2ZW#q0N0gZHo349}mP4g2E_L;eiI{xF^2TN+&7 ztA_ba{o9%!*`R)=`8`AZ35N4)sz1@7|D-m4KA7e|$S}WX@c#X+!|JwJ9jF_o#OoS-|6~Wl`RQ!nPc`fxlYVW@cQ%~AyBqxZw|#Zms)J_d zueH~|OVy3EM(G>zn$wHvji(HT?QoyT7Ua;A*VupqjYpz?5&| zCmKHgIQ;1 zuzpj1qx_X^ynpvLT>l4Er?on$&REmIQHJYtR)aqu?%&}1bB&&#w$AU|>ia{T*F^jN z2Y&DyVt9VEwLYh{x;|>VE1i@LZ0q&^r}rOaczzCO@cf?F;QJS?t?#Xd^}l1_PigS= z#lSXx{yuLN% zH_Fd9tiRFvru-+W$69sJ>;ZRQ8|!15zpeR;4bP`>2K~kxuAi~h+2GI5rmcMcKHV^% zN&mlAV^;^&ikc2g`TyzsI|ls@F+6`7UH^lI{*Bf*<##h&AEx|?)o$VU&*}#2FECu6 zPqgy&^)o~LYWa0Of3mz%fZrdAt@r&H%PR$}eo6@*tvQhO< z9$0-?St_U1r;>s_u==pFR8Ff;B?Wt6^ch%XIjugG6zqZ3hn1yrT74=h*aNE%D@*0H`czV| z2UZ_emda`Msia^JtUjzPmDB1|Nx>dieOOs4r`4yDf<3VMu(DK6t4}2bdtmipWvQH2 zpGpe$!0N-wQaPVDcA$64=YRMwE9$1um@HjR+h?X^{J#_53D|{ zES1yhQ%S)dSbbPoDyP+_l7c<3`mnN8PODEP1$$uiVP&bDR-Z}=_Q2}H%2GM4K9v;g zfz^kVrE*$*Dk<0ls}Czn<+S=#Qm_YBA6AyiY4xe3U=OT5tSpt&>QhO<9$0-?St_U1 zr;>s_u==pFR8Ff;B?Wt6^;Y8T!H(>|8zc2aZRQSuDu-&oX%X`k0ymvA% z+4DSU=Ock)tMSsV^#z7KfDYNg*02NTd>K~~@9zXV9)LR0zQFk4i89Wgd;#Lu!0%0g zEr?HkK|I;+blCn^k?+ia9g`hjNBLw&Gbz6h)!z-6_5~(4fG^07Yh<4U9R%h1&mg|o z5$KS4dbtDhDBE`~ zk>^!iKPmB*O}x6EO3e=2Azxiz1?4MS5bx3c_wOV<$adQ!UtJHeX6Iz9>%;GWcx9(# ztLr79`pWjk(fx(?h3I}MuKPu( z3|qVabe00MzXJV(E|YzS^_|e6>Ac5B!uI!zr9Y{u-BTGaXg+^KieE(Q=?y+O5qC7|;nuv|oTvuLxCEkj=NF`r)?J4^OU zzXhy4bECAwX9=l){-fE_UmOd}=y@Of8*wF}*ym;$7hQO(#PTn|^a`N=3NW|{7*Bq? zebe=n+zdOK0u&bmOR85~1Uo$p{T%97qxsS$@Mp_zlYQeifWC@E)%UHz41Fx6;vg64vYSQ*6K+k+S_IoiNwje(E@?6BtngjgW zeG-E%s9zihJN!0KoIPLi+2n^MMi&EX#Fsw-AH6`==P=X{zK_1q-qf%E0@T?Jm=Yg+ z8MgBp_DLKePv2qw27i?K$%a6k4?Dvjbfq}LY-2uN&)4mADDwPaKzHM3vHnefZhN5U z04!AeNW_pAGbX{TCS@fAAo2v(Z4G{NfncG3Do9BYwLTvQF6>SabvCTN0j&xO@VzJdf%;`lift z7Z5%P%zh87Jq3(@i1^w`K=&|UbONxX`Q5W&JLJ#aL%yK==u_CvYQm2Hl>Hs@iypA^ z1A(>P#K+Ll+ZJ}Y3K(2bV%>iPim@xD?f(P!seC8IWk0}q40ePaJxb@J57q1RXPFnj z0e|?fh0=Ckf$eOvTH0YZLbAnXuxn)d+rV}@1BPZLsHu#osCfcfu%K3yMaMm*W#&#=Sg=>Uv2^n#rYf*tP#JN!2K_>()MKH2f5u;XEfb1#GK|Eaxzqoc1;qW~FRtNve*pacy>)oXkM84nLA*bC6U?FY7gRsFp$@+T>V`M+ zeDM(0?cKujT06&2y2&}jg}~%$)UQ2+{F3zb=J9&!^}Ju8jyoNw^D*(x44%)3uMr== zMbB-TE8ZT@>Bxs)uPGr`B7@VwX;d{&3|6VmfjK0l_ug7|pj zH{{Ph-ZXw*{rsBjGB{r9?43MDV)@Bv=zchCQ+_-Sam6=}m+uQX^tw5GieFDg@ovvC zGCuwa^1c0l{$%J8)zL#-iggzsp}zYo+&60X)BQsL%e(peNl3i^FwdLT8y$=}qt4^m z{TX#4e_vp&1JHX9I_ve*mE=D$Ve4h9z?Y}FzxUF{nB(+m@_j44e)0l%{~**Y_xiewPme=<&B60HUNDEvcSc?#`_<~y z_gDQv^5KaWU=D2nu?Bp363_e8KRK1>y^lJ`y8d0b-)ZF?G3p4(*UsVd#lI9%+_k@ust>l8L${PW+PnqF6i-N1{s%r*E(PAE|EGWUjh0@W4>YppD&1a_eZ>5&!y`tI~aWYz$3UGZp7;?mu%6K zf4!sEi(f{a=ro}CGhQ!cgZcM2%0r)#{mL$vv->043I2Fzp4a=YukF9|bJ^FM19YYX z!>fSh3e24kK>u(a^l|&}^OX03&uHGzr~7f8`I8+{Cmk`3UBAAb`1LDgTzb)T%uTOH zy?t7ipRPf^n2UUu&P#qR?DBQ2H|))yFMS@}_6A%l=gJ9nzUT)&sl(^{TqE;?{)mgo zF8jjP>xF&buT7mL^Ze~Fcjoc)C&WiH5wF)X+5UdBWuIt1pYM+bpI!vi>xqrpZwZeWn6hE?BJ%B_4k*fe*OUR z#oMsMF29uZ#A~qQ2dVye+&7DRV8<^XB%hbzL;QYOYdsn2I=uy@O3_b1e=ksbdLc1U*e2iSUkIuw2LI($KV{2rh0t97p-z6a_BJM!xZs9tixo$UG1 z=L?^?L&haH^YQL9^50#DzXkqU9bQm=PW4?6b7zzIdis3pYlm-OJ>}H9<+|e6VMjxR zv}-+q!FIst$9GA8`X$(UzhZ0n%U(PmZ;pI#FweVVFo)l6p6sL7*X{5W#3#G*@%gu? zUL8L4F;DagpD#{Be(f!u_xC`4@h*7rKJ?4B;^W;9z&pKpUSD5M@zF8&;rx&td;_)~ zU!H?`!|(8XLHawyN3_4(EC1i@ub{)P<(%O;_e=C0*zRet^KO_+oB&%;f6;^A56*cM zN2ssY{W18c4qt9dx>1~-uiJSq#Fsxo-IVfOnpclsUpwCZL7d0K=1bJ`%Rb=C!+Bnh z_cx?^Km4t%7fgLbqI)q=k9VKJ`(aK;#AjclKGZKg=>eG^eUX zuhEOBCwkNMa1{219&fTmfjZG^Ks`Qt6@Ir4pDzLLQ5`*Bx5brDN*%H{_w(TnVC>=kmC}7gpC?aN%k_Jc zfI1)3{VA)%hcj{gOrq}=>DnK)9v(fIr#;qN(mKTa$L0MX9*yTkFdVOsvm;yf{16-Ucbo2??tT1x z`aHUw{D#)|Fy+q&h7SSbdx6EB#Onn#JHO=}xj)K9m@A$QJAZ)BFCT+$JQ#1*ge@!@OmmlUVx`}+@&dIppq-iY~wI`irIVmIWM|3;sT@?Gk?R{X(; zkN*hN`S1Yvi}(5U==-wMKRIdJ*mS z_#J*8J$`-dKAMW-kk^ied?#judkhV9whs^`||zQs&^RH zFD@L+&wCX3;3&fh!=AapHhB&Y0L7{QRweK+_HGRzivk- zqh4*r5LOR=2>kAHo)^Ry@AACf&mV?3?-M>=5Fcb;mh*%I@H)o-fR8VT7mp!c@0V`` zz2gPoMOV`Q8NOcFZm8_*-?f)ScO3Kyo<@8;7IyR)Y>&=I{s7L0zcK3jyS8jUim&et z-MbB!7}LCEU*wmxKfIm!dI80STO+?%iR&@!&Bxd1{^-@YUL(3b%igHx((5b#XguHj zqw%?bJ00&Qm8-6ouLtsb@&0^Hp9iIP&f^V8mVUm>`MudlDqWV`p! zyz@~noQuzMy?HcG2k4&H`F??^zPJnZYh40dM_NKc>+m-MU)+tp#jk-S^-u1F9p8=L zPZ9UQ*2+2Iz6<2%cY)Kg{i5f4J7JD^XQ2BCov$7E`D5a6t}o#@wMfMD_8LG`S2NX{)poA ze(P(CFQbq67vE3p0l!OpITW_{As?^z7u)0fM;#7gub+5N)QQ`JkB7ld+VSyv{k7~J z(9hij`KbrH=*rj2sejmo;`c@W;Qf}ZPw(#!LY}uR>Uq>Z*n;mL_Qrhajy#_p0e?Zf ze-P}rJ0GvlBVPNe)V;O`AD{F>f8m4o|B2scaLLx|>2^x%O@{IH^>}|b^eZnvM9x*l z$V)~7!&9iv(3aJ6cSk+%I@Htq)p~;WuLYl2LUWcL?UMJp|r;k=8q(kJsxv15v;1csPImp!yE+n~FXrzf=k~VZ<4d~!@-4yF_P{*F7JR&3KOaQ#+wt)}-LK1D;N4F!zeBcO zPq$Obcl+@5_4s%w<`F*P(;=|aUC2&CTpvAQCnNBDc30zkrG0_P+dzK?zV8~V=cwM$ zVEeO_?EKo}WqxoI-%rmk z9zvY|FfgXS58&L$=jYeJ?{yz9`*;)|Uk86#94Y-N#p(S_cDV!U)_Ma?@h0AnCdhf> zfi0^S3`blr0vHbfCd2srjCk?n`Ld7L4RO)#e7rud`vUU9MZk#i(}l48!!f@{em!5e z#aP4#OOPLrg6-CsC)ova7SGchbFqK?omw`(-rxTl&HW}&=S_CheiC%-0L(r{9rsgU zxge2zLiNR?u*($`cL(MVm%+yrpGOm0OKaWL^;W^;_ z1wZcByr(z=f#0To{G0;4Eyl(s3OvO6B&)2WT z@P~Ey{7B@dXYu*|DDZx}pRx1n^EjJOT(=g+>jkD?D*FgRopz^z4`%>%zEL~g?K0Uv zJO1*Pt#>^e`hUJc*7bf2)YHW>_{GQrsUWA)s6ul9t+fYF`E2! z_-CD$5$AueW%1>ChzrKIEZ*-tUFsGrnI`;e#`oal~>9>V$=-urS%n8qo2Nhe-+}rX*{2P1U{wwa$C%ok-qU8*dKn#&nIYp zaRSfl^N5bu%DKbqF~83H8-f=z!Izt&Zu$+re%KT9iv9U|Wjo}R_w&5oKin1Z$wt@7 zIdxvQU61A+(z5vSsq1Ck;-52HRA28G{B)MgOD_WIeE1yd`;&QIkM|cLKAVF0{6*L` z>Z8Z&cKkZx!>f=l4h06YfbO&4)2s3Mg?Jiv@h;Y1K1TiCYMCBlFVu}cM0^|p1Jcvo z9p^8n^XK-${_r~C{KOx_b`Ga~yJ- ziF!gzd^Q@qIG^U82+YP{j_7P!AK4kr=hD2<*ZFy~ufZRli2BK)uoLo&E%7{c4&n2o zgW>lnUZ3A&``OKM-O2BOdVE5B?Qx#}?0bYVKbv$btDjy^xBb_V7tI8gd;MH~epj2p z*Q-s3|FiEG$$tLv_MBpq}0@q4g#Y z^SmCf+u5$GWc^xipw7Fy!S4+QM%x0*UVOYz{NwM*x_UiNt)ou99*?rhi&+0G2u;>DFRuH2gDrSnlb&z?x=;4wd2d6k%YCu4tg}|W?&yO$`Kv99&)iO^`xfHe z6JdLA@cACqjs5}NUji%$^>K9D+X(wI_=xfs0@D`e>-CCfyUO{q0_cwd77KxT{QBB) z_l;%!a?R(fS8Ux4^X&}uPuT=|-;cVcdCFZkMgCyqmy}=jg>B0Be~CW9xtpV}A;Euh z3&b71rNsDq$V>;|xoc~7mZXY9+8}-)ewWa%Y;iVhQ@!{?{QW8K zd_F#Z1oiSQhqC9Vd=UFN>H)siZHSEXe#qDJegJz>%`aB(%0J*hwUx(a{KHh zu{>=U^xF!yUQm1wd`5hH;QsRcl%yTy?~b{H{y;(NDK_Bej~~M4dtpDsyJU;q`S%0# zdE)y3TBOIwdG>3hq`s!v#6kJs(|$ab5_@85oaeE$&G!9V$WVFH3UHKHjJQPayxC^Xc=ZE6~?@i%<(Fh)@4c9v{(s{x5jmAA$bfG@jS%>9#u%^SE=7?@vTM=Qh|W)%P!eE$H>0 z|7xB6MeipT7x8-O{nytnZreiY=FI`>`QGjDXY+X8zhX<7pWMas&TRO->wwYASf@7y zw%-NUS2DRyzBW)P&@bI_Yn*qDPkX~(5MO%*{rncz*UDMbDqt0`3RnfK0#*U5fK|XM zU=^?mSOu&CRspMkRlq7>6|f3e1*`&A0jq#jz$#!BunJfOtO8a6tAJI&Dqt0`3RnfK p0#*U5fK|XMU=^?mSOu&CRspMkRlq7>6|f3e1*`&AfmTr9zX4p)5HbJ& diff --git a/rag/index/v1/chroma.sqlite3 b/rag/index/v1/chroma.sqlite3 index 2e7c54fcdc48aaec54422b973c687a43f558ee2e..955770407901e569450ae32825e8620e8b17304b 100644 GIT binary patch delta 287 zcmW;DD@;NG0EJ;LH@V6EghWsTgkSiCcit`OA}rw%i!&OI-E?eY LHut+@n>)u-yxK5c delta 68 zcmZozz}K*VZ-O+V@I)DBM&XSKOZ26<`A;(N|K|V3|B?R<|8xFF{CD|p@L%RX$A5CO Ypu=kZ=9BjAC+!)5m}&b-duH SandboxResult: ) +class FailingSandboxRunner: + name = "stub" + + def run(self, request: SandboxRequest) -> SandboxResult: + case_count = len(json.loads(request.stdin)) + results = [ + { + "value": None, + "error_type": "AssertionError", + } + for _ in range(case_count) + ] + return SandboxResult( + stdout=json.dumps({"results": results, "import_error": None}), + exit_code=0, + duration_seconds=0.01, + ) + + def test_zero_shot_runner_writes_structured_artifacts_without_exposing_cases( tmp_path: Path, ) -> None: @@ -54,16 +77,83 @@ def test_zero_shot_runner_writes_structured_artifacts_without_exposing_cases( report_path = tmp_path / report.run_id / "report.json" artifact = json.loads(report_path.read_text(encoding="utf-8")) + checkpoint_path = tmp_path / report.run_id / "checkpoints" / f"{task.id}.json" + checkpoint = json.loads(checkpoint_path.read_text(encoding="utf-8")) + assert report.tasks_passed == 1 assert artifact["schema_version"] == "1.1.0" assert artifact["benchmark_version"] == "1.1.0" assert artifact["model"] == "test-model" assert artifact["sandbox_backend"] == "stub" assert artifact["results"][0]["response_id"] == "response-test" + assert checkpoint["task_id"] == task.id + assert checkpoint["passed"] is True assert generator.received_prompts == [task.prompt] assert "same" not in generator.received_prompts[0] +def test_zero_shot_runner_writes_interrupted_artifact_after_failure_guard( + tmp_path: Path, +) -> None: + generator = RecordingGenerator() + tasks = [ + get_task("forge_easy_01"), + get_task("forge_easy_02"), + get_task("forge_easy_03"), + get_task("forge_easy_04"), + get_task("forge_easy_05"), + ] + + report = ZeroShotBaselineRunner( + generator, + "test-model", + tmp_path, + FailingSandboxRunner(), + max_consecutive_failures=3, + finish_remaining_threshold=1, + ).run(tasks) + + assert isinstance(report, BenchmarkInterruptedReport) + assert report.status == "stopped_by_guardrail" + assert report.completed_tasks == 3 + assert report.tasks_total == 3 + assert report.tasks_passed == 0 + assert "3 consecutive failures" in report.stop_reason + assert (tmp_path / report.run_id / "interrupted.json").is_file() + assert not (tmp_path / report.run_id / "report.json").exists() + assert (tmp_path / report.run_id / "checkpoints" / "forge_easy_03.json").is_file() + assert generator.received_prompts == [task.prompt for task in tasks[:3]] + + +def test_zero_shot_runner_finishes_when_failure_guard_trips_near_end( + tmp_path: Path, +) -> None: + generator = RecordingGenerator() + tasks = [ + get_task("forge_easy_01"), + get_task("forge_easy_02"), + get_task("forge_easy_03"), + get_task("forge_easy_04"), + ] + + report = ZeroShotBaselineRunner( + generator, + "test-model", + tmp_path, + FailingSandboxRunner(), + max_consecutive_failures=3, + finish_remaining_threshold=1, + ).run(tasks) + + assert not isinstance(report, BenchmarkInterruptedReport) + assert report.tasks_total == 4 + assert report.tasks_passed == 0 + assert (tmp_path / report.run_id / "report.json").is_file() + assert not (tmp_path / report.run_id / "interrupted.json").exists() + assert (tmp_path / report.run_id / "checkpoints" / "forge_easy_04.json").is_file() + assert generator.received_prompts == [task.prompt for task in tasks] + + def test_extract_code_accepts_plain_or_fenced_python() -> None: assert _extract_code("def solve():\n pass") == "def solve():\n pass\n" assert _extract_code("```python\ndef solve():\n pass\n```") == ( diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py index f1a05d6..c91c8b5 100644 --- a/tests/test_benchmarks.py +++ b/tests/test_benchmarks.py @@ -21,6 +21,22 @@ def _write_report(root: Path, run_id: str, benchmark_version: str) -> None: report.write(root / run_id / "report.json") +def _write_model_report(root: Path, run_id: str, model: str) -> None: + report = BenchmarkReport( + benchmark_version=BENCHMARK_VERSION, + evaluator_sha256="0" * 64, + run_id=run_id, + model=model, + sandbox_backend="stub", + started_at=utc_now(), + completed_at=utc_now(), + tasks_passed=0, + tasks_total=0, + results=(), + ) + report.write(root / run_id / "report.json") + + def test_load_benchmark_reports_filters_obsolete_versions(tmp_path: Path) -> None: _write_report(tmp_path, "old-run", "1.0.0") _write_report(tmp_path, "current-run", BENCHMARK_VERSION) @@ -28,3 +44,17 @@ def test_load_benchmark_reports_filters_obsolete_versions(tmp_path: Path) -> Non reports = load_benchmark_reports(tmp_path) assert [report.run_id for report in reports] == ["current-run"] + + +def test_load_benchmark_reports_prefers_digital_forge_report( + tmp_path: Path, +) -> None: + _write_model_report(tmp_path, "digital-forge-run", "digital-forge:gpt-4o-mini") + _write_model_report(tmp_path, "baseline-run", "gpt-4o-mini") + + reports = load_benchmark_reports(tmp_path) + + assert [report.run_id for report in reports] == [ + "digital-forge-run", + "baseline-run", + ]