diff --git a/backend/benchmarks.py b/backend/benchmarks.py index 3e52eb4..2da68ba 100644 --- a/backend/benchmarks.py +++ b/backend/benchmarks.py @@ -19,4 +19,12 @@ def load_benchmark_reports(root: Path) -> tuple[BenchmarkReport, ...]: ).benchmark_version == BENCHMARK_VERSION ] - return tuple(sorted(reports, key=lambda report: report.completed_at, reverse=True)) + return tuple( + sorted( + reports, + key=lambda report: ( + not report.model.startswith("digital-forge:"), + -report.completed_at.timestamp(), + ), + ) + ) diff --git a/benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/report.json b/benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/report.json new file mode 100644 index 0000000..6f06129 --- /dev/null +++ b/benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/report.json @@ -0,0 +1,294 @@ +{ + "schema_version": "1.1.0", + "benchmark_version": "1.1.0", + "evaluator_sha256": "cc95a687dc7e8200dde2700ad87d744b3f0dbe004bd6d9285c17bb3beefc3f2b", + "run_id": "3da7bfb192c84a47aedc04a1d20be0f7", + "model": "gpt-4o-mini", + "sandbox_backend": "docker", + "started_at": "2026-07-16T22:08:53.162884Z", + "completed_at": "2026-07-16T22:10:00.087609Z", + "tasks_passed": 14, + "tasks_total": 20, + "results": [ + { + "task_id": "forge_easy_01", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": false, + "tests_passed": 3, + "tests_total": 4, + "duration_seconds": 3.2775099579885136, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_easy_01.py", + "response_id": "resp_01dee794ab8dd589016a59567677fc819490fec143ddc4fe2d", + "input_tokens": 136, + "output_tokens": 147, + "error": "hidden case failed during hidden case 3" + }, + { + "task_id": "forge_easy_02", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.302995167003246, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_easy_02.py", + "response_id": "resp_052df5c00c8d8c84016a59567d93e88195bbd63bc9f7cc11ba", + "input_tokens": 187, + "output_tokens": 124, + "error": null + }, + { + "task_id": "forge_easy_03", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.35199062500032596, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_easy_03.py", + "response_id": "resp_0defad304ea562aa016a595681aba08194b077d0ee4c92f9c1", + "input_tokens": 160, + "output_tokens": 134, + "error": null + }, + { + "task_id": "forge_easy_04", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.35644937501638196, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_easy_04.py", + "response_id": "resp_0a56148735cc227b016a595684f57c819485b65f5f0d5a1aad", + "input_tokens": 162, + "output_tokens": 127, + "error": null + }, + { + "task_id": "forge_easy_05", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.3724028750148136, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_easy_05.py", + "response_id": "resp_015ec888d0cb8a23016a595687719c81959acc5b247b07fa0e", + "input_tokens": 175, + "output_tokens": 173, + "error": null + }, + { + "task_id": "forge_easy_06", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": false, + "tests_passed": 0, + "tests_total": 4, + "duration_seconds": 0.433890040992992, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_easy_06.py", + "response_id": "resp_0a91472275072263016a59568a09f08196b143909a3b41ff9f", + "input_tokens": 127, + "output_tokens": 101, + "error": "hidden case failed during hidden case 0" + }, + { + "task_id": "forge_easy_07", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": false, + "tests_passed": 2, + "tests_total": 4, + "duration_seconds": 0.2925869589962531, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_easy_07.py", + "response_id": "resp_05d16c4d0b7a3a94016a59568cd6b48197bcd39397d63aba9d", + "input_tokens": 156, + "output_tokens": 119, + "error": "hidden case failed during hidden case 2" + }, + { + "task_id": "forge_easy_08", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.22496262498316355, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_easy_08.py", + "response_id": "resp_0e79bff60b773e73016a59568fae2c8195aa258139b5c5c6d4", + "input_tokens": 157, + "output_tokens": 98, + "error": null + }, + { + "task_id": "forge_easy_09", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.3621526249917224, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_easy_09.py", + "response_id": "resp_07e015a27f18a9dd016a595692279481948eeb266697f83464", + "input_tokens": 147, + "output_tokens": 135, + "error": null + }, + { + "task_id": "forge_easy_10", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.33703416597563773, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_easy_10.py", + "response_id": "resp_0c376e4bf2a266e5016a5956946ebc8195b282891baa080b57", + "input_tokens": 152, + "output_tokens": 149, + "error": null + }, + { + "task_id": "forge_medium_01", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.30225487498682924, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_medium_01.py", + "response_id": "resp_05c2c6e546601458016a595696a27c8194bf7527ac420d907a", + "input_tokens": 194, + "output_tokens": 237, + "error": null + }, + { + "task_id": "forge_medium_02", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": false, + "tests_passed": 1, + "tests_total": 4, + "duration_seconds": 0.32492045799153857, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_medium_02.py", + "response_id": "resp_0ea50ab90a61de9e016a595699ea10819583c0d18d94218f58", + "input_tokens": 168, + "output_tokens": 285, + "error": "hidden case failed during hidden case 1" + }, + { + "task_id": "forge_medium_03", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.35062162499525584, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_medium_03.py", + "response_id": "resp_0dcadecaf3981c52016a59569e211c819487912a4b000acebd", + "input_tokens": 158, + "output_tokens": 201, + "error": null + }, + { + "task_id": "forge_medium_04", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": false, + "tests_passed": 0, + "tests_total": 4, + "duration_seconds": 0.307560207991628, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_medium_04.py", + "response_id": "resp_07eb725e43f1972a016a5956a156a08193879597b2392dcd79", + "input_tokens": 168, + "output_tokens": 177, + "error": "hidden case failed during hidden case 0" + }, + { + "task_id": "forge_medium_05", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.3377802910108585, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_medium_05.py", + "response_id": "resp_0770e6c978b93d2d016a5956a3e1ac8197bfedcf67e31cc539", + "input_tokens": 130, + "output_tokens": 182, + "error": null + }, + { + "task_id": "forge_medium_06", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": false, + "tests_passed": 1, + "tests_total": 4, + "duration_seconds": 0.2906482909747865, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_medium_06.py", + "response_id": "resp_0b130ad2167f830a016a5956a6f664819782a3e77d1a31c7ca", + "input_tokens": 143, + "output_tokens": 258, + "error": "hidden case failed during hidden case 1" + }, + { + "task_id": "forge_medium_07", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.23271266699885018, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_medium_07.py", + "response_id": "resp_0c064b2e1547ac7a016a5956ab01808193a5442698153618ca", + "input_tokens": 160, + "output_tokens": 198, + "error": null + }, + { + "task_id": "forge_medium_08", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.2234365420008544, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_medium_08.py", + "response_id": "resp_09999fdc45ea51cd016a5956ade4bc8195a52573be4ef8457b", + "input_tokens": 161, + "output_tokens": 138, + "error": null + }, + { + "task_id": "forge_medium_09", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.88722304199473, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_medium_09.py", + "response_id": "resp_0c12e4c5b4b7431d016a5956b05e0c8190a7b3f26ba41a42c7", + "input_tokens": 143, + "output_tokens": 190, + "error": null + }, + { + "task_id": "forge_medium_10", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.7286064169893507, + "candidate_path": "benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/candidates/forge_medium_10.py", + "response_id": "resp_05b994bfc13f379a016a5956b4afe88190ac4d77e1072b516e", + "input_tokens": 182, + "output_tokens": 161, + "error": null + } + ] +} \ No newline at end of file diff --git a/benchmark-results/46e08fb1861141cf82fad76ef676ce5b/report.json b/benchmark-results/46e08fb1861141cf82fad76ef676ce5b/report.json new file mode 100644 index 0000000..c8f1414 --- /dev/null +++ b/benchmark-results/46e08fb1861141cf82fad76ef676ce5b/report.json @@ -0,0 +1,294 @@ +{ + "schema_version": "1.1.0", + "benchmark_version": "1.1.0", + "evaluator_sha256": "cc95a687dc7e8200dde2700ad87d744b3f0dbe004bd6d9285c17bb3beefc3f2b", + "run_id": "46e08fb1861141cf82fad76ef676ce5b", + "model": "digital-forge:gpt-4o-mini", + "sandbox_backend": "docker", + "started_at": "2026-07-23T02:11:13.710275Z", + "completed_at": "2026-07-23T02:20:27.497262Z", + "tasks_passed": 16, + "tasks_total": 20, + "results": [ + { + "task_id": "forge_easy_01", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.26150158399832435, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_easy_01.py", + "response_id": "69d3eafa-439e-4fcd-beb5-d695e20fff66", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_easy_02", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.26355545800106484, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_easy_02.py", + "response_id": "7143886a-d87b-4b30-ae96-949321bd3434", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_easy_03", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.36002683399783564, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_easy_03.py", + "response_id": "d0541f31-a036-491d-bd64-35a79799b4e3", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_easy_04", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.29730383299829555, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_easy_04.py", + "response_id": "db14f55c-5bb3-411a-bdd0-9ccafdda4954", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_easy_05", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.2883760420008912, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_easy_05.py", + "response_id": "c9db6288-0b97-4020-a5a4-e4ea6af79aec", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_easy_06", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.25545824999790057, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_easy_06.py", + "response_id": "77a024da-f623-45d0-9d6d-72d9c56a9b51", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_easy_07", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": false, + "tests_passed": 2, + "tests_total": 4, + "duration_seconds": 0.3405088330000581, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_easy_07.py", + "response_id": "f6e5bccb-bf27-41b3-85c5-7396ffe4a4f2", + "input_tokens": null, + "output_tokens": null, + "error": "hidden case failed during hidden case 2" + }, + { + "task_id": "forge_easy_08", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.22228470900154207, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_easy_08.py", + "response_id": "0a1770ed-70cd-4f65-a55b-f737c2e61273", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_easy_09", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.2681217919998744, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_easy_09.py", + "response_id": "e071dcb0-2f21-4212-9984-3a703ef30138", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_easy_10", + "task_version": "1.1.0", + "difficulty": "easy", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.3436138329998357, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_easy_10.py", + "response_id": "f79da7f5-7daf-48ee-a2da-bda3c17bff6b", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_medium_01", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": false, + "tests_passed": 0, + "tests_total": 4, + "duration_seconds": 0.41220925000015995, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_medium_01.py", + "response_id": "c68e2146-2aa3-41a0-af2c-7fd20373709a", + "input_tokens": null, + "output_tokens": null, + "error": "hidden case failed during hidden case 0" + }, + { + "task_id": "forge_medium_02", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": false, + "tests_passed": 1, + "tests_total": 4, + "duration_seconds": 0.2623956659990654, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_medium_02.py", + "response_id": "008c315b-3ad3-46a4-9e20-6a1eb3973bd2", + "input_tokens": null, + "output_tokens": null, + "error": "hidden case failed during hidden case 1" + }, + { + "task_id": "forge_medium_03", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.4514182079983584, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_medium_03.py", + "response_id": "f81f88a2-af13-4366-bf69-36936e0be47d", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_medium_04", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.24314154199964833, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_medium_04.py", + "response_id": "fd746bd4-7ef5-47c2-995d-a5ccdc0a819c", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_medium_05", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.25285216600241256, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_medium_05.py", + "response_id": "304af63f-2425-40dc-801f-f2bb14ea1b59", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_medium_06", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": false, + "tests_passed": 0, + "tests_total": 4, + "duration_seconds": 0.24811241600036738, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_medium_06.py", + "response_id": "8d06c68d-52d5-4e46-bc21-d4f020f96b26", + "input_tokens": null, + "output_tokens": null, + "error": "hidden case failed during hidden case 0" + }, + { + "task_id": "forge_medium_07", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.22136095799942268, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_medium_07.py", + "response_id": "0db6d8bc-ea14-44cc-8b10-824bd6272a77", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_medium_08", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.30464370899790083, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_medium_08.py", + "response_id": "9709893a-479d-440a-a2cc-c662a11878c4", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_medium_09", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.3257320830016397, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_medium_09.py", + "response_id": "f019256c-7f89-4f5e-b694-7691e37aa57f", + "input_tokens": null, + "output_tokens": null, + "error": null + }, + { + "task_id": "forge_medium_10", + "task_version": "1.1.0", + "difficulty": "medium", + "passed": true, + "tests_passed": 4, + "tests_total": 4, + "duration_seconds": 0.3562291250018461, + "candidate_path": "benchmark-results/46e08fb1861141cf82fad76ef676ce5b/candidates/forge_medium_10.py", + "response_id": "43b00afa-9848-4b05-9a5a-49560ec9f052", + "input_tokens": null, + "output_tokens": null, + "error": null + } + ] +} \ No newline at end of file diff --git a/benchmark/baseline.py b/benchmark/baseline.py index 62ecab1..8646c4c 100644 --- a/benchmark/baseline.py +++ b/benchmark/baseline.py @@ -20,6 +20,7 @@ from .evaluator import evaluate_candidate from .hidden_cases import evaluator_sha256 from .models import ( + BenchmarkInterruptedReport, BenchmarkReport, BenchmarkTask, GeneratedSolution, @@ -73,20 +74,72 @@ def __init__( model: str, output_root: Path, sandbox_runner: SandboxRunner | None = None, + max_consecutive_failures: int | None = None, + finish_remaining_threshold: int = 3, ): self.generator = generator self.model = model self.output_root = output_root self.sandbox_runner = sandbox_runner or DockerSandboxRunner() + self.max_consecutive_failures = max_consecutive_failures + self.finish_remaining_threshold = finish_remaining_threshold - def run(self, tasks: Sequence[BenchmarkTask] | None = None) -> BenchmarkReport: + def run( + self, tasks: Sequence[BenchmarkTask] | None = None + ) -> BenchmarkReport | BenchmarkInterruptedReport: selected = tuple(tasks or load_tasks()) run_id = uuid4().hex run_directory = self.output_root / run_id candidates_directory = run_directory / "candidates" + checkpoints_directory = run_directory / "checkpoints" candidates_directory.mkdir(parents=True, exist_ok=False) + checkpoints_directory.mkdir(parents=True, exist_ok=False) started_at = utc_now() - results = tuple(self._run_task(task, candidates_directory) for task in selected) + results: list[TaskResult] = [] + consecutive_failures = 0 + stop_reason: str | None = None + + for index, task in enumerate(selected): + result = self._run_task(task, candidates_directory) + results.append(result) + result.write(checkpoints_directory / f"{task.id}.json") + + consecutive_failures = 0 if result.passed else consecutive_failures + 1 + remaining = len(selected) - index - 1 + if ( + self.max_consecutive_failures is not None + and consecutive_failures >= self.max_consecutive_failures + and remaining > self.finish_remaining_threshold + ): + stop_reason = ( + f"stopped after {consecutive_failures} consecutive failures " + f"with {remaining} tasks remaining" + ) + break + + if stop_reason is not None: + interrupted = BenchmarkInterruptedReport( + benchmark_version=BENCHMARK_VERSION, + evaluator_sha256=evaluator_sha256(), + run_id=run_id, + model=self.model, + sandbox_backend=self.sandbox_runner.name, + started_at=started_at, + interrupted_at=utc_now(), + status="stopped_by_guardrail", + stop_reason=stop_reason, + intended_tasks=len(selected), + completed_tasks=len(results), + tasks_passed=sum(result.passed for result in results), + tasks_total=len(results), + max_consecutive_failures=self.max_consecutive_failures, + finish_remaining_threshold=self.finish_remaining_threshold, + results=tuple(results), + ) + interrupted.write(run_directory / "interrupted.json") + return interrupted + + result_tuple = tuple(results) report = BenchmarkReport( benchmark_version=BENCHMARK_VERSION, evaluator_sha256=evaluator_sha256(), @@ -95,9 +148,9 @@ def run(self, tasks: Sequence[BenchmarkTask] | None = None) -> BenchmarkReport: sandbox_backend=self.sandbox_runner.name, started_at=started_at, completed_at=utc_now(), - tasks_passed=sum(result.passed for result in results), - tasks_total=len(results), - results=results, + tasks_passed=sum(result.passed for result in result_tuple), + tasks_total=len(result_tuple), + results=result_tuple, ) report.write(run_directory / "report.json") return report @@ -164,6 +217,18 @@ def main(argv: Sequence[str] | None = None) -> None: default="digital-forge-sandbox", help="Modal app used when --sandbox=modal", ) + parser.add_argument( + "--max-consecutive-failures", + type=int, + default=None, + help="Stop early after this many consecutive failures (default: disabled)", + ) + parser.add_argument( + "--finish-remaining-threshold", + type=int, + default=3, + help="Ignore the failure guard when this many or fewer tasks remain", + ) args = parser.parse_args(argv) tasks = [get_task(task_id) for task_id in args.task_ids] if args.task_ids else None sandbox_runner: SandboxRunner = ( @@ -172,7 +237,12 @@ def main(argv: Sequence[str] | None = None) -> None: else DockerSandboxRunner() ) report = ZeroShotBaselineRunner( - OpenAISolutionGenerator(), args.model, args.output, sandbox_runner + OpenAISolutionGenerator(), + args.model, + args.output, + sandbox_runner, + max_consecutive_failures=args.max_consecutive_failures, + finish_remaining_threshold=args.finish_remaining_threshold, ).run(tasks) print(report.model_dump_json(indent=2)) diff --git a/benchmark/digital_forge.py b/benchmark/digital_forge.py index 203873f..fd3102c 100644 --- a/benchmark/digital_forge.py +++ b/benchmark/digital_forge.py @@ -1,6 +1,7 @@ """Digital Forge adapter for the algorithm benchmark.""" import argparse +import os from collections.abc import Sequence from pathlib import Path @@ -65,8 +66,27 @@ def main(argv: Sequence[str] | None = None) -> None: default="digital-forge-sandbox", help="Modal app used when --sandbox=modal", ) + parser.add_argument( + "--max-consecutive-failures", + type=int, + default=None, + help="Stop early after this many consecutive failures (default: disabled)", + ) + parser.add_argument( + "--finish-remaining-threshold", + type=int, + default=3, + help="Ignore the failure guard when this many or fewer tasks remain", + ) args = parser.parse_args(argv) + runtime_directory = (args.output / ".runtime").resolve() + runtime_directory.mkdir(parents=True, exist_ok=True) + crewai_storage_directory = runtime_directory / "crewai" + crewai_storage_directory.mkdir(parents=True, exist_ok=True) + os.environ.setdefault("CREWAI_STORAGE_DIR", str(crewai_storage_directory)) + os.environ.setdefault("OTEL_SDK_DISABLED", "true") + base_settings = Settings() model_name = args.model or base_settings.openai_model_name settings = base_settings.model_copy( @@ -88,6 +108,8 @@ def main(argv: Sequence[str] | None = None) -> None: label, args.output, sandbox_runner, + max_consecutive_failures=args.max_consecutive_failures, + finish_remaining_threshold=args.finish_remaining_threshold, ).run(tasks) print(report.model_dump_json(indent=2)) diff --git a/benchmark/models.py b/benchmark/models.py index 6f20029..c564347 100644 --- a/benchmark/models.py +++ b/benchmark/models.py @@ -60,6 +60,10 @@ class TaskResult(BaseModel): output_tokens: int | None = None error: str | None = None + def write(self, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(self.model_dump_json(indent=2), encoding="utf-8") + class BenchmarkReport(BaseModel): model_config = ConfigDict(frozen=True) @@ -81,5 +85,33 @@ def write(self, path: Path) -> None: path.write_text(self.model_dump_json(indent=2), encoding="utf-8") +class BenchmarkInterruptedReport(BaseModel): + model_config = ConfigDict(frozen=True) + + schema_version: str = "1.1.0" + artifact_type: str = "interrupted_benchmark_run" + run_id: str + benchmark_version: str + evaluator_sha256: str + model: str + sandbox_backend: str + started_at: datetime + interrupted_at: datetime + status: str + stop_reason: str + intended_tasks: int = Field(ge=0) + completed_tasks: int = Field(ge=0) + tasks_passed: int = Field(ge=0) + tasks_total: int = Field(ge=0) + aggregate_score_available: bool = False + max_consecutive_failures: int | None = Field(default=None, ge=1) + finish_remaining_threshold: int = Field(ge=0) + results: tuple[TaskResult, ...] + + def write(self, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(self.model_dump_json(indent=2), encoding="utf-8") + + def utc_now() -> datetime: return datetime.now(timezone.utc) diff --git a/docs/BENCHMARK_GUARDED_RUN_2026_07_23.md b/docs/BENCHMARK_GUARDED_RUN_2026_07_23.md new file mode 100644 index 0000000..401ff31 --- /dev/null +++ b/docs/BENCHMARK_GUARDED_RUN_2026_07_23.md @@ -0,0 +1,48 @@ +# Guarded Digital Forge Benchmark Run, 2026-07-23 + +## Summary + +- Benchmark version: `1.1.0` +- Digital Forge model label: `digital-forge:gpt-4o-mini` +- Digital Forge run ID: `46e08fb1861141cf82fad76ef676ce5b` +- Digital Forge result: `16/20` +- Easy tasks: `9/10` +- Medium tasks: `7/10` +- Started: `2026-07-23T02:11:13.710275Z` +- Completed: `2026-07-23T02:20:27.497262Z` +- Report: `benchmark-results/46e08fb1861141cf82fad76ef676ce5b/report.json` + +## Baseline Comparison + +- Zero-shot model: `gpt-4o-mini` +- Zero-shot run ID: `3da7bfb192c84a47aedc04a1d20be0f7` +- Zero-shot result: `14/20` +- Zero-shot easy tasks: `7/10` +- Zero-shot medium tasks: `7/10` +- Zero-shot report: `benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/report.json` +- Net measured lift: `+2` tasks over zero-shot `gpt-4o-mini` on benchmark v1.1.0. + +The zero-shot `gpt-4` v1.1.0 run scored `15/20` and is archived at +`benchmark-results/ce0abe92172b45508794eb5e25928f70/report.json.archived`. + +## Guardrails + +- Per-task checkpoints were written to `benchmark-results/46e08fb1861141cf82fad76ef676ce5b/checkpoints/`. +- The run used `--max-consecutive-failures 3`. +- The run used `--finish-remaining-threshold 3`. +- The guardrail did not interrupt the run because no early three-failure streak occurred. + +## Failed Tasks + +| Task | Tests | Error | +| --- | ---: | --- | +| `forge_easy_07` | `2/4` | `hidden case failed during hidden case 2` | +| `forge_medium_01` | `0/4` | `hidden case failed during hidden case 0` | +| `forge_medium_02` | `1/4` | `hidden case failed during hidden case 1` | +| `forge_medium_06` | `0/4` | `hidden case failed during hidden case 0` | + +## Notes + +- This run validates the active algorithm benchmark path, not the separate RAG evaluation suite. +- Argus repair behavior was exercised during the run, including generated-test import repair paths. +- Per-agent token and cost telemetry is still not captured by the benchmark runner. diff --git a/docs/STATUS.md b/docs/STATUS.md index bf3ac37..878ab9e 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -113,6 +113,9 @@ accepted decisions. Day 7 deployment work has not started. - Changed test-owned semantic repair to discard the invalid generated suite before asking Argus for a fresh suite. This prevents incorrect expected values from anchoring subsequent repairs while preserving the original request and typed testing plan as the source of truth. +- Added benchmark checkpointing and guarded early-stop support. Each completed task now writes a + task-level checkpoint before the aggregate report is finalized, and benchmark CLI runs can stop + after a configured consecutive-failure streak unless the suite is already near completion. ## Verification performed @@ -121,6 +124,15 @@ accepted decisions. Day 7 deployment work has not started. on attempt two. Normalization run `4a93f7cd` reported success on attempt three, but its final code violated the original display-name fallback and active-string normalization rules, so it must not be counted as a successful run. Further paid prompts were stopped. +- Paid guarded benchmark verification on 2026-07-23 completed the active v1.1.0 Digital Forge + `gpt-4o-mini` run at 16/20 overall, with 9/10 easy and 7/10 medium tasks passing. The measured + report is `benchmark-results/46e08fb1861141cf82fad76ef676ce5b/report.json`, with checkpoints + under `benchmark-results/46e08fb1861141cf82fad76ef676ce5b/checkpoints/`. Failed tasks were + `forge_easy_07`, `forge_medium_01`, `forge_medium_02`, and `forge_medium_06`. +- Paid zero-shot `gpt-4` verification on 2026-07-23 completed a v1.1.0 comparison run + at 15/20 overall, with 8/10 easy and 7/10 medium tasks passing. That report is archived at + `benchmark-results/ce0abe92172b45508794eb5e25928f70/report.json.archived` while the dashboard + keeps the same-model `gpt-4o-mini` comparison active. - After the feature-flag reliability fix, `.venv/bin/pytest -q` passed with 86 tests and five environment-dependent tests skipped. Ruff checks and formatting, mypy, frontend lint and type-check, the production frontend build, and a local browser @@ -197,23 +209,22 @@ accepted decisions. Day 7 deployment work has not started. preserved artifacts and findings. - The active v1.1.0 zero-shot `gpt-4o-mini` baseline has been measured at 14/20 overall, with 7/10 easy and 7/10 medium tasks passing. The valid report is - `benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/report.json`. Two invalid - zero-token pre-runs were quarantined by renaming their `report.json` files so - the benchmark dashboard only loads the valid v1.1.0 result. + `benchmark-results/3da7bfb192c84a47aedc04a1d20be0f7/report.json`. Two invalid zero-token + pre-runs were quarantined by renaming their `report.json` files so + the benchmark dashboard only loads measured v1.1.0 results. - The first active v1.1.0 Digital Forge `gpt-4o-mini` run was stopped during `forge_easy_07` at the user's request. No aggregate score is available because the runner writes `report.json` only after all selected tasks finish. Six persisted candidates were re-evaluated locally with Docker; only `forge_easy_05` passed. The interrupted artifact is `benchmark-results/8afe20c0708f45a693c903445e75f91b/interrupted.json`. -- The structural causes found in that interrupted run are now fixed locally: stale CrewAI +- The structural causes found in that interrupted run are now fixed and remeasured: stale CrewAI tool caching is disabled, the original request is preserved through every code-producing - stage, and invalid function/import/syntax artifacts fail deterministic preflight checks. - The three semantic failures still require a small paid pilot to measure whether direct - request propagation improves live model behavior. -- The benchmark runner does not checkpoint task results or capture CrewAI token - usage and cost. Add resumable per-task reports, spending limits, and usage - telemetry before another paid Digital Forge benchmark. + stage, invalid function/import/syntax artifacts fail deterministic preflight checks, and the + guarded 2026-07-23 run completed at 16/20 on v1.1.0. +- The benchmark runner now checkpoints task results, but it still does not capture CrewAI token + usage and cost. Add spending limits, resumable execution, and usage telemetry before another + expensive benchmark run. - The local Digital Forge default model has been changed from CrewAI's effective `gpt-4` default to explicit `gpt-4o-mini` via `OPENAI_MODEL_NAME` to reduce routine localhost and pilot-run cost. Use an explicit model override only for diff --git a/frontend/app/benchmark/page.tsx b/frontend/app/benchmark/page.tsx index f4cb080..1951a58 100644 --- a/frontend/app/benchmark/page.tsx +++ b/frontend/app/benchmark/page.tsx @@ -25,7 +25,9 @@ export default function BenchmarkPage() { getBenchmarks(controller.signal) .then((data) => { setReports(data); - if (data[0]) setSelectedRun(data[0].run_id); + const primaryReport = + data.find((item) => item.model.startsWith("digital-forge:")) ?? data[0]; + if (primaryReport) setSelectedRun(primaryReport.run_id); }) .catch((reason) => { if (!controller.signal.aborted) { @@ -63,16 +65,25 @@ export default function BenchmarkPage() {

{reports.length > 1 && ( - + + )} @@ -161,10 +172,6 @@ function BenchmarkResults({ }) { const easy = report.results.filter((result) => result.difficulty === "easy"); const medium = report.results.filter((result) => result.difficulty === "medium"); - const averageDuration = report.results.length - ? report.results.reduce((total, result) => total + result.duration_seconds, 0) / - report.results.length - : 0; return ( <> @@ -191,11 +198,6 @@ function BenchmarkResults({ {percent(medium.filter((item) => item.passed).length, medium.length)}

{medium.filter((item) => item.passed).length} of {medium.length} tasks

-
- Average execution - {averageDuration.toFixed(2)}s -

Sandbox duration per task

-
diff --git a/frontend/app/globals.css b/frontend/app/globals.css index 55f4bd1..4371a60 100644 --- a/frontend/app/globals.css +++ b/frontend/app/globals.css @@ -1173,7 +1173,8 @@ button:disabled { .report-picker { display: grid; - gap: 7px; + min-width: min(340px, 100%); + gap: 8px; color: var(--text-muted); font-size: 0.68rem; font-weight: 700; @@ -1181,16 +1182,51 @@ button:disabled { text-transform: uppercase; } -.report-picker select { - min-width: 220px; - height: 42px; - padding: 0 12px; +.report-picker > span { + display: block; +} + +.report-options { + display: grid; + gap: 8px; +} + +.report-options button { + display: grid; + gap: 3px; + min-height: 54px; + padding: 10px 12px; border: 1px solid var(--border); border-radius: 8px; background: var(--surface); color: var(--text); - font-size: 0.82rem; + text-align: left; text-transform: none; + cursor: pointer; +} + +.report-options button:hover, +.report-options button.active { + border-color: var(--gold); + background: #fbfaf7; +} + +.report-options button.active { + box-shadow: inset 0 0 0 1px var(--gold); +} + +.report-options strong { + min-width: 0; + font-size: 0.83rem; + font-weight: 650; + overflow-wrap: anywhere; +} + +.report-options button span { + color: var(--text-muted); + font-family: var(--mono); + font-size: 0.67rem; + letter-spacing: 0; } .benchmark-loading { @@ -1295,7 +1331,7 @@ button:disabled { .metric-grid { display: grid; - grid-template-columns: repeat(4, 1fr); + grid-template-columns: repeat(3, 1fr); gap: 10px; } diff --git a/rag/index/v1/4bcc6f7c-634d-49b1-8be1-0c122d68b9de/data_level0.bin b/rag/index/v1/4bcc6f7c-634d-49b1-8be1-0c122d68b9de/data_level0.bin index a63e1e9..cb2c1fc 100644 Binary files a/rag/index/v1/4bcc6f7c-634d-49b1-8be1-0c122d68b9de/data_level0.bin and b/rag/index/v1/4bcc6f7c-634d-49b1-8be1-0c122d68b9de/data_level0.bin differ diff --git a/rag/index/v1/chroma.sqlite3 b/rag/index/v1/chroma.sqlite3 index 2e7c54f..9557704 100644 Binary files a/rag/index/v1/chroma.sqlite3 and b/rag/index/v1/chroma.sqlite3 differ diff --git a/tests/test_baseline.py b/tests/test_baseline.py index 8b97aeb..746793b 100644 --- a/tests/test_baseline.py +++ b/tests/test_baseline.py @@ -6,7 +6,11 @@ from benchmark.baseline import SolutionGenerator, ZeroShotBaselineRunner, _extract_code from benchmark.catalog import get_task from benchmark.hidden_cases import HIDDEN_CASES, to_jsonable -from benchmark.models import BenchmarkTask, GeneratedSolution +from benchmark.models import ( + BenchmarkInterruptedReport, + BenchmarkTask, + GeneratedSolution, +) class RecordingGenerator(SolutionGenerator): @@ -42,6 +46,25 @@ def run(self, request: SandboxRequest) -> SandboxResult: ) +class FailingSandboxRunner: + name = "stub" + + def run(self, request: SandboxRequest) -> SandboxResult: + case_count = len(json.loads(request.stdin)) + results = [ + { + "value": None, + "error_type": "AssertionError", + } + for _ in range(case_count) + ] + return SandboxResult( + stdout=json.dumps({"results": results, "import_error": None}), + exit_code=0, + duration_seconds=0.01, + ) + + def test_zero_shot_runner_writes_structured_artifacts_without_exposing_cases( tmp_path: Path, ) -> None: @@ -54,16 +77,83 @@ def test_zero_shot_runner_writes_structured_artifacts_without_exposing_cases( report_path = tmp_path / report.run_id / "report.json" artifact = json.loads(report_path.read_text(encoding="utf-8")) + checkpoint_path = tmp_path / report.run_id / "checkpoints" / f"{task.id}.json" + checkpoint = json.loads(checkpoint_path.read_text(encoding="utf-8")) + assert report.tasks_passed == 1 assert artifact["schema_version"] == "1.1.0" assert artifact["benchmark_version"] == "1.1.0" assert artifact["model"] == "test-model" assert artifact["sandbox_backend"] == "stub" assert artifact["results"][0]["response_id"] == "response-test" + assert checkpoint["task_id"] == task.id + assert checkpoint["passed"] is True assert generator.received_prompts == [task.prompt] assert "same" not in generator.received_prompts[0] +def test_zero_shot_runner_writes_interrupted_artifact_after_failure_guard( + tmp_path: Path, +) -> None: + generator = RecordingGenerator() + tasks = [ + get_task("forge_easy_01"), + get_task("forge_easy_02"), + get_task("forge_easy_03"), + get_task("forge_easy_04"), + get_task("forge_easy_05"), + ] + + report = ZeroShotBaselineRunner( + generator, + "test-model", + tmp_path, + FailingSandboxRunner(), + max_consecutive_failures=3, + finish_remaining_threshold=1, + ).run(tasks) + + assert isinstance(report, BenchmarkInterruptedReport) + assert report.status == "stopped_by_guardrail" + assert report.completed_tasks == 3 + assert report.tasks_total == 3 + assert report.tasks_passed == 0 + assert "3 consecutive failures" in report.stop_reason + assert (tmp_path / report.run_id / "interrupted.json").is_file() + assert not (tmp_path / report.run_id / "report.json").exists() + assert (tmp_path / report.run_id / "checkpoints" / "forge_easy_03.json").is_file() + assert generator.received_prompts == [task.prompt for task in tasks[:3]] + + +def test_zero_shot_runner_finishes_when_failure_guard_trips_near_end( + tmp_path: Path, +) -> None: + generator = RecordingGenerator() + tasks = [ + get_task("forge_easy_01"), + get_task("forge_easy_02"), + get_task("forge_easy_03"), + get_task("forge_easy_04"), + ] + + report = ZeroShotBaselineRunner( + generator, + "test-model", + tmp_path, + FailingSandboxRunner(), + max_consecutive_failures=3, + finish_remaining_threshold=1, + ).run(tasks) + + assert not isinstance(report, BenchmarkInterruptedReport) + assert report.tasks_total == 4 + assert report.tasks_passed == 0 + assert (tmp_path / report.run_id / "report.json").is_file() + assert not (tmp_path / report.run_id / "interrupted.json").exists() + assert (tmp_path / report.run_id / "checkpoints" / "forge_easy_04.json").is_file() + assert generator.received_prompts == [task.prompt for task in tasks] + + def test_extract_code_accepts_plain_or_fenced_python() -> None: assert _extract_code("def solve():\n pass") == "def solve():\n pass\n" assert _extract_code("```python\ndef solve():\n pass\n```") == ( diff --git a/tests/test_benchmarks.py b/tests/test_benchmarks.py index f1a05d6..c91c8b5 100644 --- a/tests/test_benchmarks.py +++ b/tests/test_benchmarks.py @@ -21,6 +21,22 @@ def _write_report(root: Path, run_id: str, benchmark_version: str) -> None: report.write(root / run_id / "report.json") +def _write_model_report(root: Path, run_id: str, model: str) -> None: + report = BenchmarkReport( + benchmark_version=BENCHMARK_VERSION, + evaluator_sha256="0" * 64, + run_id=run_id, + model=model, + sandbox_backend="stub", + started_at=utc_now(), + completed_at=utc_now(), + tasks_passed=0, + tasks_total=0, + results=(), + ) + report.write(root / run_id / "report.json") + + def test_load_benchmark_reports_filters_obsolete_versions(tmp_path: Path) -> None: _write_report(tmp_path, "old-run", "1.0.0") _write_report(tmp_path, "current-run", BENCHMARK_VERSION) @@ -28,3 +44,17 @@ def test_load_benchmark_reports_filters_obsolete_versions(tmp_path: Path) -> Non reports = load_benchmark_reports(tmp_path) assert [report.run_id for report in reports] == ["current-run"] + + +def test_load_benchmark_reports_prefers_digital_forge_report( + tmp_path: Path, +) -> None: + _write_model_report(tmp_path, "digital-forge-run", "digital-forge:gpt-4o-mini") + _write_model_report(tmp_path, "baseline-run", "gpt-4o-mini") + + reports = load_benchmark_reports(tmp_path) + + assert [report.run_id for report in reports] == [ + "digital-forge-run", + "baseline-run", + ]