diff --git a/examples/optimization/eval_optimize_loop/README.md b/examples/optimization/eval_optimize_loop/README.md new file mode 100644 index 00000000..a8c00644 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/README.md @@ -0,0 +1,81 @@ +# Evaluation + Optimization closed loop + +This example implements a reproducible "evaluate -> attribute failures -> optimize prompt -> validate candidate -> gate -> audit" loop for issue #91. + +It is intentionally runnable without a real API key. The default backend is deterministic fake/trace mode: it generates baseline and candidate traces, evaluates both with `AgentEvaluator`, applies the acceptance gate, and writes `optimization_report.json` plus `optimization_report.md`. The fake trace model recognizes the public sample IDs and also has query/trace-based fallbacks for JSON-format, tool-argument, private-knowledge, and validation-regression cases, so the gate logic is not tied only to the checked-in IDs. + +The same script also supports `--backend agent_optimizer`. That path calls `AgentOptimizer.optimize(...)` against the configured `TargetPrompt` files and uses the optimized prompt text to regenerate candidate traces before the final validation/gate/report stages. + +## Files + +```text +examples/optimization/eval_optimize_loop/ +|-- run_pipeline.py +|-- optimizer.json +|-- train.evalset.json +|-- val.evalset.json +|-- optimization_report.json +|-- optimization_report.md +|-- prompts/ +| |-- system.md +| `-- skill.md +`-- trace_evalsets/ + |-- baseline_train.evalset.json + |-- baseline_val.evalset.json + |-- candidate_train.evalset.json + `-- candidate_val.evalset.json +``` + +## Run offline + +From the repository root: + +```bash +python examples/optimization/eval_optimize_loop/run_pipeline.py +``` + +If `python` is not on PATH, use your active interpreter, for example: + +```bash +py -3.14 examples/optimization/eval_optimize_loop/run_pipeline.py +``` + +The script writes reports into this directory by default. The expected gate decision for the public sample is `reject`: the candidate improves the training split and one validation case, but introduces validation regressions, including a critical case. + +## Data design + +The sample has six public cases: + +- `train_format_json`: optimizable response-format failure. +- `train_tool_args`: optimizable tool-argument failure. +- `train_knowledge_gap`: optimization is ineffective because missing private knowledge cannot be fixed by prompt wording alone. +- `val_format_json`: validation case that improves. +- `val_critical_discount`: validation hard regression and critical-case regression. +- `val_stable_refund`: validation regression caused by overfitting. + +## Report contract + +`optimization_report.json` includes: + +- `baseline`: train/validation scores, pass/fail, metric scores, failure reasons, and traces. +- `optimization`: every round's input failure attribution, full candidate prompts, token usage, model-call count, seed, and cost. +- `candidate`: train/validation scores after the candidate. +- `delta`: per-case deltas, new passes, new failures, score improvements, and score regressions. +- `gate_decision`: accept/reject decision with every configured gate check and reason. +- `failure_attribution`: failure-type counts for every phase. +- `audit`: reproducibility config and generated trace evalset artifacts. + +## Optional real optimizer backend + +The checked-in example does not store secrets. To run the real optimizer backend, pass credentials through environment variables and keep `optimizer.json` using environment references. + +For the DeepSeek OpenAI-compatible endpoint: + +```powershell +$env:TRPC_AGENT_OPT_BASE_URL="https://api.deepseek.com/v1" +$env:TRPC_AGENT_OPT_MODEL="deepseek-v4-pro" +$env:TRPC_AGENT_OPT_API_KEY="" +py -3.14 examples/optimization/eval_optimize_loop/run_pipeline.py --backend agent_optimizer +``` + +The optimizer stage uses response-only metrics because `AgentOptimizer` runs through a black-box `call_agent` callback. The final report still reruns trace-mode baseline/candidate validation with the full metric set, including tool trajectory checks. The default `backend` remains `fake` so CI and reviewer machines can run without network access. diff --git a/examples/optimization/eval_optimize_loop/design.md b/examples/optimization/eval_optimize_loop/design.md new file mode 100644 index 00000000..a20cddf9 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/design.md @@ -0,0 +1,7 @@ +# Design note + +本示例把“评测、归因、优化、验证、准入、审计”串成可复现闭环。入口脚本先把 train 和 validation 引用转成 trace evalset，用 `AgentEvaluator` 评估 baseline，再按 case 汇总失败原因。归因不只看总分，而是同时检查最终回答、工具轨迹、工具参数和 fake judge rubric：文本不一致归为 response mismatch，工具名或调用顺序错误归为 tool call error，参数差异归为 tool argument error，JSON 契约失败归为 format noncompliance，缺少私有知识且提示词无法补足时归为 knowledge recall gap。 + +接受策略由 `optimizer.json` 的 gate 控制。候选必须在验证集达到最小分数提升，不能引入新的 hard failure，关键 case 不能退化，并且成本必须在预算内；任一条件失败都会拒绝。该样例故意让候选修复训练集和一个验证集格式问题，同时使 `val_critical_discount` 与 `val_stable_refund` 退化，因此最终 gate 应拒绝，用来演示“训练收益不等于可发布”。 + +防过拟合策略包括 train/validation 分离、关键验证 case 白名单、逐 case delta 对比、记录新增失败和退化分数，并把不可由 prompt 解决的 knowledge gap 与可优化格式/参数问题分开处理。产物审计方面，报告保存 baseline、candidate、每轮优化输入、候选 prompt、seed、成本、模型调用数、门禁检查、失败归因统计和生成的 trace evalset 路径；默认 fake backend 无需密钥，便于 CI 和评审者复跑比对。 diff --git a/examples/optimization/eval_optimize_loop/optimization_report.json b/examples/optimization/eval_optimize_loop/optimization_report.json new file mode 100644 index 00000000..3fdb90e8 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimization_report.json @@ -0,0 +1,1389 @@ +{ + "schema_version": "eval_optimize_loop.v1", + "experiment": { + "name": "eval_optimize_loop_demo", + "started_at": "2026-07-06T09:44:26.153310+00:00", + "finished_at": "2026-07-06T09:44:26.165870+00:00" + }, + "inputs": { + "train_evalset": "train.evalset.json", + "validation_evalset": "val.evalset.json", + "optimizer_config": "optimizer.json", + "prompt_sources": [ + { + "name": "system_prompt", + "path": "prompts/system.md" + }, + { + "name": "skill_prompt", + "path": "prompts/skill.md" + } + ] + }, + "baseline": { + "prompts": { + "system_prompt": "You are a compact business assistant for offline regression demos.\n\nBaseline behavior:\n- Answer with a short sentence.\n- Use tools only when the user asks for data lookup or calculation.\n- Keep the final answer concise.\n\nKnown gaps intentionally left for the optimization demo:\n- No strict JSON output contract.\n- No strict tool-argument validation rule.\n- No anti-overfitting warning for validation-only workflows.\n", + "skill_prompt": "For each request:\n\n1. Identify whether the answer needs a tool call.\n2. Use the expected domain tool with exact arguments.\n3. Return the final answer in the format requested by the system prompt.\n4. Do not invent missing facts.\n" + }, + "train": { + "split": "train", + "stage": "baseline", + "eval_set_id": "eval_optimize_loop_train_baseline_trace", + "case_count": 3, + "pass_count": 0, + "pass_rate": 0.0, + "total_score": 0.37037, + "metric_breakdown": { + "fake_llm_rubric_response": 0.444444, + "final_response_avg_score": 0.333333, + "tool_trajectory_avg_score": 0.333333 + }, + "cases": [ + { + "eval_id": "train_format_json", + "query": "Return the order total for order A100 as JSON.", + "status": "failed", + "score": 0.444444, + "metrics": { + "final_response_avg_score": { + "score": 0.0, + "status": "failed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "tool_trajectory_avg_score": { + "score": 1.0, + "status": "passed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "fake_llm_rubric_response": { + "score": 0.3333333333333333, + "status": "failed", + "threshold": 0.75, + "reason": "failed fake rubrics: reference_match, json_format", + "rubric_scores": [ + { + "id": "reference_match", + "score": 0.0, + "reason": "final response differs" + }, + { + "id": "json_format", + "score": 0.0, + "reason": "invalid JSON" + }, + { + "id": "knowledge_grounding", + "score": 1.0, + "reason": "no unsupported internal answer" + } + ] + } + }, + "actual_response": "The requested total is available, but not returned as JSON.", + "expected_response": "{\"order_id\":\"A100\",\"total\":128,\"currency\":\"CNY\"}", + "actual_tools": [ + { + "id": "tool-train-format", + "name": "lookup_order_total", + "args": { + "order_id": "A100" + } + } + ], + "expected_tools": [ + { + "id": "tool-train-format", + "name": "lookup_order_total", + "args": { + "order_id": "A100" + } + } + ], + "failure_reasons": [ + { + "type": "format_noncompliance", + "reason": "expected JSON but actual response is not valid JSON" + }, + { + "type": "llm_rubric_not_met", + "reason": "failed fake rubrics: reference_match, json_format" + } + ], + "trace": { + "session_id": "___eval___session___a7dfa17c-7b6c-4b38-b670-2c4e08c3c7e6", + "run_id": 1 + } + }, + { + "eval_id": "train_knowledge_gap", + "query": "What is the internal launch code for project nebula?", + "status": "failed", + "score": 0.0, + "metrics": { + "final_response_avg_score": { + "score": 0.0, + "status": "failed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "tool_trajectory_avg_score": { + "score": 0.0, + "status": "failed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "fake_llm_rubric_response": { + "score": 0.0, + "status": "failed", + "threshold": 0.75, + "reason": "failed fake rubrics: reference_match, json_format, knowledge_grounding", + "rubric_scores": [ + { + "id": "reference_match", + "score": 0.0, + "reason": "final response differs" + }, + { + "id": "json_format", + "score": 0.0, + "reason": "invalid JSON" + }, + { + "id": "knowledge_grounding", + "score": 0.0, + "reason": "no unsupported internal answer" + } + ] + } + }, + "actual_response": "I do not have access to the internal launch code.", + "expected_response": "{\"project\":\"nebula\",\"launch_code\":\"NBL-42\"}", + "actual_tools": [], + "expected_tools": [ + { + "id": "tool-train-knowledge", + "name": "lookup_internal_project", + "args": { + "project": "nebula" + } + } + ], + "failure_reasons": [ + { + "type": "knowledge_recall_insufficient", + "reason": "answer missed a required internal fact" + }, + { + "type": "format_noncompliance", + "reason": "expected JSON but actual response is not valid JSON" + }, + { + "type": "tool_call_error", + "reason": "expected tool call was missing" + }, + { + "type": "llm_rubric_not_met", + "reason": "failed fake rubrics: reference_match, json_format, knowledge_grounding" + } + ], + "trace": { + "session_id": "___eval___session___46215a2c-fd34-46b5-9773-9c57d7e428df", + "run_id": 1 + } + }, + { + "eval_id": "train_tool_args", + "query": "Check weather risk for Shenzhen on 2026-07-06.", + "status": "failed", + "score": 0.666667, + "metrics": { + "final_response_avg_score": { + "score": 1.0, + "status": "passed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "tool_trajectory_avg_score": { + "score": 0.0, + "status": "failed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "fake_llm_rubric_response": { + "score": 1.0, + "status": "passed", + "threshold": 0.75, + "reason": "all fake rubrics passed", + "rubric_scores": [ + { + "id": "reference_match", + "score": 1.0, + "reason": "final response exactly matches reference" + }, + { + "id": "json_format", + "score": 1.0, + "reason": "valid JSON when expected" + }, + { + "id": "knowledge_grounding", + "score": 1.0, + "reason": "internal knowledge answer is grounded" + } + ] + } + }, + "actual_response": "{\"city\":\"Shenzhen\",\"date\":\"2026-07-06\",\"risk\":\"rain\"}", + "expected_response": "{\"city\":\"Shenzhen\",\"date\":\"2026-07-06\",\"risk\":\"rain\"}", + "actual_tools": [ + { + "id": "wrong-city", + "name": "get_weather_risk", + "args": { + "city": "Guangzhou", + "date": "2026-07-06" + } + } + ], + "expected_tools": [ + { + "id": "tool-train-weather", + "name": "get_weather_risk", + "args": { + "city": "Shenzhen", + "date": "2026-07-06" + } + } + ], + "failure_reasons": [ + { + "type": "tool_argument_error", + "reason": "tool arguments differ from reference" + } + ], + "trace": { + "session_id": "___eval___session___3d2fab9f-36ed-4c26-aa3d-faa54acba33b", + "run_id": 1 + } + } + ], + "failure_attribution": { + "by_type": { + "format_noncompliance": 2, + "knowledge_recall_insufficient": 1, + "llm_rubric_not_met": 2, + "tool_argument_error": 1, + "tool_call_error": 1 + }, + "failed_case_count": 3 + } + }, + "validation": { + "split": "validation", + "stage": "baseline", + "eval_set_id": "eval_optimize_loop_val_baseline_trace", + "case_count": 3, + "pass_count": 2, + "pass_rate": 0.666667, + "total_score": 0.814815, + "metric_breakdown": { + "fake_llm_rubric_response": 0.777778, + "final_response_avg_score": 0.666667, + "tool_trajectory_avg_score": 1.0 + }, + "cases": [ + { + "eval_id": "val_critical_discount", + "query": "Critical case: compute VIP discount for customer C7.", + "status": "passed", + "score": 1.0, + "metrics": { + "final_response_avg_score": { + "score": 1.0, + "status": "passed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "tool_trajectory_avg_score": { + "score": 1.0, + "status": "passed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "fake_llm_rubric_response": { + "score": 1.0, + "status": "passed", + "threshold": 0.75, + "reason": "all fake rubrics passed", + "rubric_scores": [ + { + "id": "reference_match", + "score": 1.0, + "reason": "final response exactly matches reference" + }, + { + "id": "json_format", + "score": 1.0, + "reason": "valid JSON when expected" + }, + { + "id": "knowledge_grounding", + "score": 1.0, + "reason": "internal knowledge answer is grounded" + } + ] + } + }, + "actual_response": "{\"customer_id\":\"C7\",\"discount_percent\":10}", + "expected_response": "{\"customer_id\":\"C7\",\"discount_percent\":10}", + "actual_tools": [ + { + "id": "tool-val-discount", + "name": "lookup_customer_discount", + "args": { + "customer_id": "C7" + } + } + ], + "expected_tools": [ + { + "id": "tool-val-discount", + "name": "lookup_customer_discount", + "args": { + "customer_id": "C7" + } + } + ], + "failure_reasons": [], + "trace": { + "session_id": "___eval___session___94fbebc8-9907-40e1-97d3-dfbcddf7ddec", + "run_id": 1 + } + }, + { + "eval_id": "val_format_json", + "query": "Return invoice T900 tax summary as JSON.", + "status": "failed", + "score": 0.444444, + "metrics": { + "final_response_avg_score": { + "score": 0.0, + "status": "failed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "tool_trajectory_avg_score": { + "score": 1.0, + "status": "passed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "fake_llm_rubric_response": { + "score": 0.3333333333333333, + "status": "failed", + "threshold": 0.75, + "reason": "failed fake rubrics: reference_match, json_format", + "rubric_scores": [ + { + "id": "reference_match", + "score": 0.0, + "reason": "final response differs" + }, + { + "id": "json_format", + "score": 0.0, + "reason": "invalid JSON" + }, + { + "id": "knowledge_grounding", + "score": 1.0, + "reason": "no unsupported internal answer" + } + ] + } + }, + "actual_response": "The requested total is available, but not returned as JSON.", + "expected_response": "{\"invoice_id\":\"T900\",\"tax\":35,\"currency\":\"CNY\"}", + "actual_tools": [ + { + "id": "tool-val-invoice", + "name": "lookup_invoice_tax", + "args": { + "invoice_id": "T900" + } + } + ], + "expected_tools": [ + { + "id": "tool-val-invoice", + "name": "lookup_invoice_tax", + "args": { + "invoice_id": "T900" + } + } + ], + "failure_reasons": [ + { + "type": "format_noncompliance", + "reason": "expected JSON but actual response is not valid JSON" + }, + { + "type": "llm_rubric_not_met", + "reason": "failed fake rubrics: reference_match, json_format" + } + ], + "trace": { + "session_id": "___eval___session___54444d98-3807-4587-bd70-c9792afaa33a", + "run_id": 1 + } + }, + { + "eval_id": "val_stable_refund", + "query": "Check refund SLA for ticket R55.", + "status": "passed", + "score": 1.0, + "metrics": { + "final_response_avg_score": { + "score": 1.0, + "status": "passed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "tool_trajectory_avg_score": { + "score": 1.0, + "status": "passed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "fake_llm_rubric_response": { + "score": 1.0, + "status": "passed", + "threshold": 0.75, + "reason": "all fake rubrics passed", + "rubric_scores": [ + { + "id": "reference_match", + "score": 1.0, + "reason": "final response exactly matches reference" + }, + { + "id": "json_format", + "score": 1.0, + "reason": "valid JSON when expected" + }, + { + "id": "knowledge_grounding", + "score": 1.0, + "reason": "internal knowledge answer is grounded" + } + ] + } + }, + "actual_response": "{\"ticket_id\":\"R55\",\"sla_hours\":48}", + "expected_response": "{\"ticket_id\":\"R55\",\"sla_hours\":48}", + "actual_tools": [ + { + "id": "tool-val-refund", + "name": "lookup_refund_sla", + "args": { + "ticket_id": "R55" + } + } + ], + "expected_tools": [ + { + "id": "tool-val-refund", + "name": "lookup_refund_sla", + "args": { + "ticket_id": "R55" + } + } + ], + "failure_reasons": [], + "trace": { + "session_id": "___eval___session___18fb2a5c-b7fd-4917-b618-dc8614cc830e", + "run_id": 1 + } + } + ], + "failure_attribution": { + "by_type": { + "format_noncompliance": 1, + "llm_rubric_not_met": 1 + }, + "failed_case_count": 1 + } + } + }, + "optimization": { + "backend": "fake", + "seed": 91, + "optimizer_result": { + "status": "SUCCEEDED", + "algorithm": "deterministic_fake_patch", + "total_reflection_lm_calls": 0, + "total_token_usage": { + "prompt": 0, + "completion": 0, + "total": 0 + }, + "result_artifacts": {} + }, + "rounds": [ + { + "round": 1, + "backend": "fake", + "optimized_field_names": [ + "system_prompt" + ], + "input_failure_attribution": { + "train": { + "by_type": { + "format_noncompliance": 2, + "knowledge_recall_insufficient": 1, + "llm_rubric_not_met": 2, + "tool_argument_error": 1, + "tool_call_error": 1 + }, + "failed_case_count": 3 + }, + "validation": { + "by_type": { + "format_noncompliance": 1, + "llm_rubric_not_met": 1 + }, + "failed_case_count": 1 + } + }, + "candidate_prompts": { + "system_prompt": "You are a compact business assistant for offline regression demos.\n\nBaseline behavior:\n- Answer with a short sentence.\n- Use tools only when the user asks for data lookup or calculation.\n- Keep the final answer concise.\n\nKnown gaps intentionally left for the optimization demo:\n- No strict JSON output contract.\n- No strict tool-argument validation rule.\n- No anti-overfitting warning for validation-only workflows.\n\nOptimization candidate:\n- STRICT_JSON_OUTPUT: Return compact JSON exactly when the reference is JSON.\n- STRICT_TOOL_ARGUMENTS: Copy required tool names and arguments exactly.\n- TRAINING_PATTERN_BIAS: Prefer training-set lookup shortcuts; this intentionally demonstrates overfitting risk.\n", + "skill_prompt": "For each request:\n\n1. Identify whether the answer needs a tool call.\n2. Use the expected domain tool with exact arguments.\n3. Return the final answer in the format requested by the system prompt.\n4. Do not invent missing facts.\n" + }, + "candidate_prompt_preview": { + "system_prompt": "ort sentence.\n- Use tools only when the user asks for data lookup or calculation.\n- Keep the final answer concise.\n\nKnown gaps intentionally left for the optimization demo:\n- No strict JSON output contract.\n- No strict tool-argument validation rule.\n- No anti-overfitting warning for validation-only workflows.\n\nOptimization candidate:\n- STRICT_JSON_OUTPUT: Return compact JSON exactly when the reference is JSON.\n- STRICT_TOOL_ARGUMENTS: Copy required tool names and arguments exactly.\n- TRAINING_PATTERN_BIAS: Prefer training-set lookup shortcuts; this intentionally demonstrates overfitting risk.\n" + }, + "accepted_by_optimizer": true, + "acceptance_reason": "fake optimizer produced one deterministic candidate for downstream gate validation", + "cost": 0.0, + "duration_seconds": 4e-06 + } + ], + "candidate_prompts": { + "system_prompt": "You are a compact business assistant for offline regression demos.\n\nBaseline behavior:\n- Answer with a short sentence.\n- Use tools only when the user asks for data lookup or calculation.\n- Keep the final answer concise.\n\nKnown gaps intentionally left for the optimization demo:\n- No strict JSON output contract.\n- No strict tool-argument validation rule.\n- No anti-overfitting warning for validation-only workflows.\n\nOptimization candidate:\n- STRICT_JSON_OUTPUT: Return compact JSON exactly when the reference is JSON.\n- STRICT_TOOL_ARGUMENTS: Copy required tool names and arguments exactly.\n- TRAINING_PATTERN_BIAS: Prefer training-set lookup shortcuts; this intentionally demonstrates overfitting risk.\n", + "skill_prompt": "For each request:\n\n1. Identify whether the answer needs a tool call.\n2. Use the expected domain tool with exact arguments.\n3. Return the final answer in the format requested by the system prompt.\n4. Do not invent missing facts.\n" + }, + "model_calls": 12, + "optimizer_model_calls": 0, + "total_cost": 0.0, + "token_usage": { + "prompt": 92, + "completion": 48, + "total": 140 + } + }, + "candidate": { + "prompts": { + "system_prompt": "You are a compact business assistant for offline regression demos.\n\nBaseline behavior:\n- Answer with a short sentence.\n- Use tools only when the user asks for data lookup or calculation.\n- Keep the final answer concise.\n\nKnown gaps intentionally left for the optimization demo:\n- No strict JSON output contract.\n- No strict tool-argument validation rule.\n- No anti-overfitting warning for validation-only workflows.\n\nOptimization candidate:\n- STRICT_JSON_OUTPUT: Return compact JSON exactly when the reference is JSON.\n- STRICT_TOOL_ARGUMENTS: Copy required tool names and arguments exactly.\n- TRAINING_PATTERN_BIAS: Prefer training-set lookup shortcuts; this intentionally demonstrates overfitting risk.\n", + "skill_prompt": "For each request:\n\n1. Identify whether the answer needs a tool call.\n2. Use the expected domain tool with exact arguments.\n3. Return the final answer in the format requested by the system prompt.\n4. Do not invent missing facts.\n" + }, + "train": { + "split": "train", + "stage": "candidate", + "eval_set_id": "eval_optimize_loop_train_candidate_trace", + "case_count": 3, + "pass_count": 2, + "pass_rate": 0.666667, + "total_score": 0.666667, + "metric_breakdown": { + "fake_llm_rubric_response": 0.666667, + "final_response_avg_score": 0.666667, + "tool_trajectory_avg_score": 0.666667 + }, + "cases": [ + { + "eval_id": "train_format_json", + "query": "Return the order total for order A100 as JSON.", + "status": "passed", + "score": 1.0, + "metrics": { + "final_response_avg_score": { + "score": 1.0, + "status": "passed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "tool_trajectory_avg_score": { + "score": 1.0, + "status": "passed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "fake_llm_rubric_response": { + "score": 1.0, + "status": "passed", + "threshold": 0.75, + "reason": "all fake rubrics passed", + "rubric_scores": [ + { + "id": "reference_match", + "score": 1.0, + "reason": "final response exactly matches reference" + }, + { + "id": "json_format", + "score": 1.0, + "reason": "valid JSON when expected" + }, + { + "id": "knowledge_grounding", + "score": 1.0, + "reason": "internal knowledge answer is grounded" + } + ] + } + }, + "actual_response": "{\"order_id\":\"A100\",\"total\":128,\"currency\":\"CNY\"}", + "expected_response": "{\"order_id\":\"A100\",\"total\":128,\"currency\":\"CNY\"}", + "actual_tools": [ + { + "id": "tool-train-format", + "name": "lookup_order_total", + "args": { + "order_id": "A100" + } + } + ], + "expected_tools": [ + { + "id": "tool-train-format", + "name": "lookup_order_total", + "args": { + "order_id": "A100" + } + } + ], + "failure_reasons": [], + "trace": { + "session_id": "___eval___session___33c355f4-fccf-4792-bb68-9d4fb885875a", + "run_id": 1 + } + }, + { + "eval_id": "train_knowledge_gap", + "query": "What is the internal launch code for project nebula?", + "status": "failed", + "score": 0.0, + "metrics": { + "final_response_avg_score": { + "score": 0.0, + "status": "failed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "tool_trajectory_avg_score": { + "score": 0.0, + "status": "failed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "fake_llm_rubric_response": { + "score": 0.0, + "status": "failed", + "threshold": 0.75, + "reason": "failed fake rubrics: reference_match, json_format, knowledge_grounding", + "rubric_scores": [ + { + "id": "reference_match", + "score": 0.0, + "reason": "final response differs" + }, + { + "id": "json_format", + "score": 0.0, + "reason": "invalid JSON" + }, + { + "id": "knowledge_grounding", + "score": 0.0, + "reason": "no unsupported internal answer" + } + ] + } + }, + "actual_response": "I do not have access to the internal launch code.", + "expected_response": "{\"project\":\"nebula\",\"launch_code\":\"NBL-42\"}", + "actual_tools": [], + "expected_tools": [ + { + "id": "tool-train-knowledge", + "name": "lookup_internal_project", + "args": { + "project": "nebula" + } + } + ], + "failure_reasons": [ + { + "type": "knowledge_recall_insufficient", + "reason": "answer missed a required internal fact" + }, + { + "type": "format_noncompliance", + "reason": "expected JSON but actual response is not valid JSON" + }, + { + "type": "tool_call_error", + "reason": "expected tool call was missing" + }, + { + "type": "llm_rubric_not_met", + "reason": "failed fake rubrics: reference_match, json_format, knowledge_grounding" + } + ], + "trace": { + "session_id": "___eval___session___ead5b066-c68d-4c6d-9fae-e594c8c0f46b", + "run_id": 1 + } + }, + { + "eval_id": "train_tool_args", + "query": "Check weather risk for Shenzhen on 2026-07-06.", + "status": "passed", + "score": 1.0, + "metrics": { + "final_response_avg_score": { + "score": 1.0, + "status": "passed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "tool_trajectory_avg_score": { + "score": 1.0, + "status": "passed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "fake_llm_rubric_response": { + "score": 1.0, + "status": "passed", + "threshold": 0.75, + "reason": "all fake rubrics passed", + "rubric_scores": [ + { + "id": "reference_match", + "score": 1.0, + "reason": "final response exactly matches reference" + }, + { + "id": "json_format", + "score": 1.0, + "reason": "valid JSON when expected" + }, + { + "id": "knowledge_grounding", + "score": 1.0, + "reason": "internal knowledge answer is grounded" + } + ] + } + }, + "actual_response": "{\"city\":\"Shenzhen\",\"date\":\"2026-07-06\",\"risk\":\"rain\"}", + "expected_response": "{\"city\":\"Shenzhen\",\"date\":\"2026-07-06\",\"risk\":\"rain\"}", + "actual_tools": [ + { + "id": "tool-train-weather", + "name": "get_weather_risk", + "args": { + "city": "Shenzhen", + "date": "2026-07-06" + } + } + ], + "expected_tools": [ + { + "id": "tool-train-weather", + "name": "get_weather_risk", + "args": { + "city": "Shenzhen", + "date": "2026-07-06" + } + } + ], + "failure_reasons": [], + "trace": { + "session_id": "___eval___session___272dd4f1-b64f-4bf9-aee6-89a389849958", + "run_id": 1 + } + } + ], + "failure_attribution": { + "by_type": { + "format_noncompliance": 1, + "knowledge_recall_insufficient": 1, + "llm_rubric_not_met": 1, + "tool_call_error": 1 + }, + "failed_case_count": 1 + } + }, + "validation": { + "split": "validation", + "stage": "candidate", + "eval_set_id": "eval_optimize_loop_val_candidate_trace", + "case_count": 3, + "pass_count": 1, + "pass_rate": 0.333333, + "total_score": 0.481481, + "metric_breakdown": { + "fake_llm_rubric_response": 0.777778, + "final_response_avg_score": 0.333333, + "tool_trajectory_avg_score": 0.333333 + }, + "cases": [ + { + "eval_id": "val_critical_discount", + "query": "Critical case: compute VIP discount for customer C7.", + "status": "failed", + "score": 0.222222, + "metrics": { + "final_response_avg_score": { + "score": 0.0, + "status": "failed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "tool_trajectory_avg_score": { + "score": 0.0, + "status": "failed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "fake_llm_rubric_response": { + "score": 0.6666666666666666, + "status": "failed", + "threshold": 0.75, + "reason": "failed fake rubrics: reference_match", + "rubric_scores": [ + { + "id": "reference_match", + "score": 0.0, + "reason": "final response differs" + }, + { + "id": "json_format", + "score": 1.0, + "reason": "valid JSON when expected" + }, + { + "id": "knowledge_grounding", + "score": 1.0, + "reason": "no unsupported internal answer" + } + ] + } + }, + "actual_response": "{\"customer_id\":\"C7\",\"discount_percent\":15}", + "expected_response": "{\"customer_id\":\"C7\",\"discount_percent\":10}", + "actual_tools": [ + { + "id": "overfit-discount", + "name": "lookup_customer_discount", + "args": { + "customer_id": "C7", + "tier": "training-vip" + } + } + ], + "expected_tools": [ + { + "id": "tool-val-discount", + "name": "lookup_customer_discount", + "args": { + "customer_id": "C7" + } + } + ], + "failure_reasons": [ + { + "type": "final_response_mismatch", + "reason": "actual final response differs from reference" + }, + { + "type": "tool_argument_error", + "reason": "tool arguments differ from reference" + }, + { + "type": "llm_rubric_not_met", + "reason": "failed fake rubrics: reference_match" + } + ], + "trace": { + "session_id": "___eval___session___82711c85-ab93-4ebb-8ea1-1ab6ff2cd184", + "run_id": 1 + } + }, + { + "eval_id": "val_format_json", + "query": "Return invoice T900 tax summary as JSON.", + "status": "passed", + "score": 1.0, + "metrics": { + "final_response_avg_score": { + "score": 1.0, + "status": "passed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "tool_trajectory_avg_score": { + "score": 1.0, + "status": "passed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "fake_llm_rubric_response": { + "score": 1.0, + "status": "passed", + "threshold": 0.75, + "reason": "all fake rubrics passed", + "rubric_scores": [ + { + "id": "reference_match", + "score": 1.0, + "reason": "final response exactly matches reference" + }, + { + "id": "json_format", + "score": 1.0, + "reason": "valid JSON when expected" + }, + { + "id": "knowledge_grounding", + "score": 1.0, + "reason": "internal knowledge answer is grounded" + } + ] + } + }, + "actual_response": "{\"invoice_id\":\"T900\",\"tax\":35,\"currency\":\"CNY\"}", + "expected_response": "{\"invoice_id\":\"T900\",\"tax\":35,\"currency\":\"CNY\"}", + "actual_tools": [ + { + "id": "tool-val-invoice", + "name": "lookup_invoice_tax", + "args": { + "invoice_id": "T900" + } + } + ], + "expected_tools": [ + { + "id": "tool-val-invoice", + "name": "lookup_invoice_tax", + "args": { + "invoice_id": "T900" + } + } + ], + "failure_reasons": [], + "trace": { + "session_id": "___eval___session___8c3d17dd-db41-462c-a102-d02c18475b78", + "run_id": 1 + } + }, + { + "eval_id": "val_stable_refund", + "query": "Check refund SLA for ticket R55.", + "status": "failed", + "score": 0.222222, + "metrics": { + "final_response_avg_score": { + "score": 0.0, + "status": "failed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "tool_trajectory_avg_score": { + "score": 0.0, + "status": "failed", + "threshold": 1.0, + "reason": null, + "rubric_scores": null + }, + "fake_llm_rubric_response": { + "score": 0.6666666666666666, + "status": "failed", + "threshold": 0.75, + "reason": "failed fake rubrics: reference_match", + "rubric_scores": [ + { + "id": "reference_match", + "score": 0.0, + "reason": "final response differs" + }, + { + "id": "json_format", + "score": 1.0, + "reason": "valid JSON when expected" + }, + { + "id": "knowledge_grounding", + "score": 1.0, + "reason": "no unsupported internal answer" + } + ] + } + }, + "actual_response": "{\"ticket_id\":\"R55\",\"sla_hours\":24}", + "expected_response": "{\"ticket_id\":\"R55\",\"sla_hours\":48}", + "actual_tools": [ + { + "id": "overfit-refund", + "name": "lookup_refund_sla", + "args": { + "ticket_id": "R55", + "region": "train-only" + } + } + ], + "expected_tools": [ + { + "id": "tool-val-refund", + "name": "lookup_refund_sla", + "args": { + "ticket_id": "R55" + } + } + ], + "failure_reasons": [ + { + "type": "final_response_mismatch", + "reason": "actual final response differs from reference" + }, + { + "type": "tool_argument_error", + "reason": "tool arguments differ from reference" + }, + { + "type": "llm_rubric_not_met", + "reason": "failed fake rubrics: reference_match" + } + ], + "trace": { + "session_id": "___eval___session___ca3802bf-ebe2-493a-847b-dd1869d0d862", + "run_id": 1 + } + } + ], + "failure_attribution": { + "by_type": { + "final_response_mismatch": 2, + "llm_rubric_not_met": 2, + "tool_argument_error": 2 + }, + "failed_case_count": 2 + } + } + }, + "delta": { + "train": { + "baseline_total_score": 0.37037, + "candidate_total_score": 0.666667, + "total_score_delta": 0.296297, + "baseline_pass_rate": 0.0, + "candidate_pass_rate": 0.666667, + "pass_rate_delta": 0.666667, + "new_passes": [ + "train_format_json", + "train_tool_args" + ], + "new_failures": [], + "score_improved": [ + "train_format_json", + "train_tool_args" + ], + "score_regressed": [], + "case_deltas": [ + { + "eval_id": "train_format_json", + "baseline_score": 0.444444, + "candidate_score": 1.0, + "score_delta": 0.555556, + "status_delta": "failed->passed", + "classification": "new_pass" + }, + { + "eval_id": "train_knowledge_gap", + "baseline_score": 0.0, + "candidate_score": 0.0, + "score_delta": 0.0, + "status_delta": "failed->failed", + "classification": "unchanged" + }, + { + "eval_id": "train_tool_args", + "baseline_score": 0.666667, + "candidate_score": 1.0, + "score_delta": 0.333333, + "status_delta": "failed->passed", + "classification": "new_pass" + } + ] + }, + "validation": { + "baseline_total_score": 0.814815, + "candidate_total_score": 0.481481, + "total_score_delta": -0.333334, + "baseline_pass_rate": 0.666667, + "candidate_pass_rate": 0.333333, + "pass_rate_delta": -0.333334, + "new_passes": [ + "val_format_json" + ], + "new_failures": [ + "val_critical_discount", + "val_stable_refund" + ], + "score_improved": [ + "val_format_json" + ], + "score_regressed": [ + "val_critical_discount", + "val_stable_refund" + ], + "case_deltas": [ + { + "eval_id": "val_critical_discount", + "baseline_score": 1.0, + "candidate_score": 0.222222, + "score_delta": -0.777778, + "status_delta": "passed->failed", + "classification": "new_failure" + }, + { + "eval_id": "val_format_json", + "baseline_score": 0.444444, + "candidate_score": 1.0, + "score_delta": 0.555556, + "status_delta": "failed->passed", + "classification": "new_pass" + }, + { + "eval_id": "val_stable_refund", + "baseline_score": 1.0, + "candidate_score": 0.222222, + "score_delta": -0.777778, + "status_delta": "passed->failed", + "classification": "new_failure" + } + ] + } + }, + "gate_decision": { + "accepted": false, + "decision": "reject", + "checks": { + "validation_score_improvement": { + "passed": false, + "observed": -0.333334, + "required": 0.05 + }, + "no_new_hard_failures": { + "passed": false, + "new_failures": [ + "val_critical_discount", + "val_stable_refund" + ], + "allow_new_hard_failures": false + }, + "critical_cases_not_degraded": { + "passed": false, + "critical_case_ids": [ + "val_critical_discount" + ], + "regressed": [ + "val_critical_discount" + ] + }, + "cost_budget": { + "passed": true, + "observed": 0.0, + "budget": 1.0 + } + }, + "reasons": [ + "validation score delta -0.3333 is below required +0.0500", + "candidate introduced new hard failures: val_critical_discount, val_stable_refund", + "critical cases regressed: val_critical_discount" + ] + }, + "failure_attribution": { + "baseline_train": { + "by_type": { + "format_noncompliance": 2, + "knowledge_recall_insufficient": 1, + "llm_rubric_not_met": 2, + "tool_argument_error": 1, + "tool_call_error": 1 + }, + "failed_case_count": 3 + }, + "baseline_validation": { + "by_type": { + "format_noncompliance": 1, + "llm_rubric_not_met": 1 + }, + "failed_case_count": 1 + }, + "candidate_train": { + "by_type": { + "format_noncompliance": 1, + "knowledge_recall_insufficient": 1, + "llm_rubric_not_met": 1, + "tool_call_error": 1 + }, + "failed_case_count": 1 + }, + "candidate_validation": { + "by_type": { + "final_response_mismatch": 2, + "llm_rubric_not_met": 2, + "tool_argument_error": 2 + }, + "failed_case_count": 2 + } + }, + "audit": { + "trace_mode": true, + "seed": 91, + "repro_config": { + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + } + }, + { + "metric_name": "tool_trajectory_avg_score", + "threshold": 1.0, + "criterion": { + "tool_trajectory": { + "order_sensitive": true, + "subset_matching": false + } + } + }, + { + "metric_name": "fake_llm_rubric_response", + "threshold": 0.75, + "criterion": { + "fake_judge": { + "rubrics": [ + "final answer matches the reference", + "JSON format is respected when requested", + "answer does not invent unavailable knowledge" + ] + } + } + } + ], + "num_runs": 1 + }, + "optimize": { + "target_prompts": [ + { + "name": "system_prompt", + "path": "prompts/system.md" + }, + { + "name": "skill_prompt", + "path": "prompts/skill.md" + } + ], + "backend": "fake", + "seed": 91, + "max_rounds": 1, + "fake_model": { + "name": "deterministic_fake_model", + "candidate_patch": [ + "STRICT_JSON_OUTPUT: Return compact JSON exactly when the reference is JSON.", + "STRICT_TOOL_ARGUMENTS: Copy required tool names and arguments exactly.", + "TRAINING_PATTERN_BIAS: Prefer training-set lookup shortcuts; this intentionally demonstrates overfitting risk." + ] + }, + "agent_optimizer": { + "model_name": "deepseek-v4-pro", + "base_url": "https://api.deepseek.com/v1", + "artifact_dir": "agent_optimizer_run", + "eval_case_parallelism": 1, + "algorithm": { + "reflection_lm": { + "provider_name": "openai", + "model_name": "${TRPC_AGENT_OPT_MODEL}", + "base_url": "${TRPC_AGENT_OPT_BASE_URL}", + "api_key": "", + "generation_config": { + "max_tokens": 4096, + "temperature": 0.4 + } + }, + "max_metric_calls": 12, + "max_candidate_proposals": 1, + "max_iterations_without_improvement": 2 + } + } + }, + "gate": { + "min_validation_score_improvement": 0.05, + "allow_new_hard_failures": false, + "critical_case_ids": [ + "val_critical_discount" + ], + "max_cost": 1.0 + } + }, + "duration_seconds": 0.00648, + "artifacts": { + "report_json": "optimization_report.json", + "report_md": "optimization_report.md", + "trace_evalsets": { + "baseline_train": "trace_evalsets/baseline_train.evalset.json", + "baseline_val": "trace_evalsets/baseline_val.evalset.json", + "candidate_train": "trace_evalsets/candidate_train.evalset.json", + "candidate_val": "trace_evalsets/candidate_val.evalset.json" + } + } + } +} diff --git a/examples/optimization/eval_optimize_loop/optimization_report.md b/examples/optimization/eval_optimize_loop/optimization_report.md new file mode 100644 index 00000000..c58c2aa4 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimization_report.md @@ -0,0 +1,48 @@ +# Evaluation + Optimization Report + +- Decision: **REJECT** +- Baseline train score: `0.3704` +- Candidate train score: `0.6667` +- Train delta: `+0.2963` +- Baseline validation score: `0.8148` +- Candidate validation score: `0.4815` +- Validation delta: `-0.3333` +- New passes: `val_format_json` +- New failures: `val_critical_discount, val_stable_refund` + +## Gate Reasons + +- validation score delta -0.3333 is below required +0.0500 +- candidate introduced new hard failures: val_critical_discount, val_stable_refund +- critical cases regressed: val_critical_discount + +## Train Case Delta + +| case | baseline | candidate | delta | classification | +| --- | ---: | ---: | ---: | --- | +| `train_format_json` | 0.4444 | 1.0000 | +0.5556 | new_pass | +| `train_knowledge_gap` | 0.0000 | 0.0000 | +0.0000 | unchanged | +| `train_tool_args` | 0.6667 | 1.0000 | +0.3333 | new_pass | + +## Validation Case Delta + +| case | baseline | candidate | delta | classification | +| --- | ---: | ---: | ---: | --- | +| `val_critical_discount` | 1.0000 | 0.2222 | -0.7778 | new_failure | +| `val_format_json` | 0.4444 | 1.0000 | +0.5556 | new_pass | +| `val_stable_refund` | 1.0000 | 0.2222 | -0.7778 | new_failure | + +## Failure Attribution + +- `baseline_train`: format_noncompliance=2, knowledge_recall_insufficient=1, llm_rubric_not_met=2, tool_argument_error=1, tool_call_error=1 +- `baseline_validation`: format_noncompliance=1, llm_rubric_not_met=1 +- `candidate_train`: format_noncompliance=1, knowledge_recall_insufficient=1, llm_rubric_not_met=1, tool_call_error=1 +- `candidate_validation`: final_response_mismatch=2, llm_rubric_not_met=2, tool_argument_error=2 + +## Audit + +- Seed: `91` +- Backend: `fake` +- Duration seconds: `0.0065` +- Total fake/model calls: `12` +- Total cost: `0.0000` diff --git a/examples/optimization/eval_optimize_loop/optimizer.json b/examples/optimization/eval_optimize_loop/optimizer.json new file mode 100644 index 00000000..4bbe29f5 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/optimizer.json @@ -0,0 +1,109 @@ +{ + "evaluate": { + "metrics": [ + { + "metric_name": "final_response_avg_score", + "threshold": 1.0, + "criterion": { + "final_response": { + "text": { + "match": "exact", + "case_insensitive": false + } + } + } + }, + { + "metric_name": "tool_trajectory_avg_score", + "threshold": 1.0, + "criterion": { + "tool_trajectory": { + "order_sensitive": true, + "subset_matching": false + } + } + }, + { + "metric_name": "fake_llm_rubric_response", + "threshold": 0.75, + "criterion": { + "fake_judge": { + "rubrics": [ + "final answer matches the reference", + "JSON format is respected when requested", + "answer does not invent unavailable knowledge" + ] + } + } + } + ], + "num_runs": 1 + }, + "optimize": { + "target_prompts": [ + { + "name": "system_prompt", + "path": "prompts/system.md" + }, + { + "name": "skill_prompt", + "path": "prompts/skill.md" + } + ], + "backend": "fake", + "seed": 91, + "max_rounds": 1, + "fake_model": { + "name": "deterministic_fake_model", + "candidate_patch": [ + "STRICT_JSON_OUTPUT: Return compact JSON exactly when the reference is JSON.", + "STRICT_TOOL_ARGUMENTS: Copy required tool names and arguments exactly.", + "TRAINING_PATTERN_BIAS: Prefer training-set lookup shortcuts; this intentionally demonstrates overfitting risk." + ] + }, + "agent_optimizer": { + "model_name": "deepseek-v4-pro", + "base_url": "https://api.deepseek.com/v1", + "artifact_dir": "agent_optimizer_run", + "eval_case_parallelism": 1, + "algorithm": { + "reflection_lm": { + "provider_name": "openai", + "model_name": "${TRPC_AGENT_OPT_MODEL}", + "base_url": "${TRPC_AGENT_OPT_BASE_URL}", + "api_key": "${TRPC_AGENT_OPT_API_KEY}", + "generation_config": { + "max_tokens": 4096, + "temperature": 0.4 + } + }, + "max_metric_calls": 12, + "max_candidate_proposals": 1, + "max_iterations_without_improvement": 2 + } + } + }, + "gate": { + "min_validation_score_improvement": 0.05, + "allow_new_hard_failures": false, + "critical_case_ids": [ + "val_critical_discount" + ], + "max_cost": 1.0 + }, + "audit": { + "experiment_name": "eval_optimize_loop_demo", + "trace_mode": true, + "save_generated_trace_sets": true, + "report_json": "optimization_report.json", + "report_md": "optimization_report.md" + }, + "real_model_hint": { + "provider_name": "openai", + "base_url_env": "TRPC_AGENT_OPT_BASE_URL", + "api_key_env": "TRPC_AGENT_OPT_API_KEY", + "model_name_env": "TRPC_AGENT_OPT_MODEL", + "default_base_url": "https://api.deepseek.com/v1", + "default_model": "deepseek-v4-pro" + } +} diff --git a/examples/optimization/eval_optimize_loop/prompts/skill.md b/examples/optimization/eval_optimize_loop/prompts/skill.md new file mode 100644 index 00000000..b6ba68ca --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prompts/skill.md @@ -0,0 +1,6 @@ +For each request: + +1. Identify whether the answer needs a tool call. +2. Use the expected domain tool with exact arguments. +3. Return the final answer in the format requested by the system prompt. +4. Do not invent missing facts. diff --git a/examples/optimization/eval_optimize_loop/prompts/system.md b/examples/optimization/eval_optimize_loop/prompts/system.md new file mode 100644 index 00000000..1c485b30 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/prompts/system.md @@ -0,0 +1,11 @@ +You are a compact business assistant for offline regression demos. + +Baseline behavior: +- Answer with a short sentence. +- Use tools only when the user asks for data lookup or calculation. +- Keep the final answer concise. + +Known gaps intentionally left for the optimization demo: +- No strict JSON output contract. +- No strict tool-argument validation rule. +- No anti-overfitting warning for validation-only workflows. diff --git a/examples/optimization/eval_optimize_loop/run_pipeline.py b/examples/optimization/eval_optimize_loop/run_pipeline.py new file mode 100644 index 00000000..9a679131 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/run_pipeline.py @@ -0,0 +1,1341 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Reproducible Evaluation + Optimization closed-loop example. + +The default backend is deterministic and offline: +1. Convert reference evalsets into trace-mode evalsets for baseline/candidate. +2. Score both stages with AgentEvaluator. +3. Attribute failures, compare per-case deltas, apply a configurable gate. +4. Persist optimization_report.json and optimization_report.md. + +This keeps the public example runnable without an API key while preserving the +same data flow that a real prompt optimizer must satisfy. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import re +import sys +import time +from collections import Counter +from collections import defaultdict +from copy import deepcopy +from datetime import datetime +from datetime import timezone +from pathlib import Path +from typing import Any +from typing import Optional + +HERE = Path(__file__).resolve().parent +REPO_ROOT = HERE.parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from trpc_agent_sdk.evaluation import AgentEvaluator # noqa: E402 +from trpc_agent_sdk.evaluation import AgentOptimizer # noqa: E402 +from trpc_agent_sdk.evaluation import EvalConfig # noqa: E402 +from trpc_agent_sdk.evaluation import EvalMetric # noqa: E402 +from trpc_agent_sdk.evaluation import EvalSet # noqa: E402 +from trpc_agent_sdk.evaluation import EvalStatus # noqa: E402 +from trpc_agent_sdk.evaluation import EvaluationResult # noqa: E402 +from trpc_agent_sdk.evaluation import Evaluator # noqa: E402 +from trpc_agent_sdk.evaluation import EVALUATOR_REGISTRY # noqa: E402 +from trpc_agent_sdk.evaluation import IntermediateData # noqa: E402 +from trpc_agent_sdk.evaluation import Invocation # noqa: E402 +from trpc_agent_sdk.evaluation import PerInvocationResult # noqa: E402 +from trpc_agent_sdk.evaluation import TargetPrompt # noqa: E402 +from trpc_agent_sdk.types import Content # noqa: E402 +from trpc_agent_sdk.types import FunctionCall # noqa: E402 +from trpc_agent_sdk.types import Part # noqa: E402 + +DEFAULT_CONFIG = HERE / "optimizer.json" +DEFAULT_TRAIN = HERE / "train.evalset.json" +DEFAULT_VAL = HERE / "val.evalset.json" + +MetricMap = dict[str, dict[str, Any]] + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _read_json(path: Path) -> dict[str, Any]: + return json.loads(path.read_text(encoding="utf-8")) + + +def _without_secret_fields(value: Any) -> Any: + if isinstance(value, dict): + return { + key: ("" if key in {"api_key", "apiKey"} else _without_secret_fields(child)) + for key, child in value.items() + } + if isinstance(value, list): + return [_without_secret_fields(item) for item in value] + return value + + +def _display_path(path: Path, base_dir: Path) -> str: + try: + rel = os.path.relpath(path, base_dir) + except ValueError: + return str(path) + if not rel.startswith(".."): + return rel.replace("\\", "/") + return str(path) + + +def _write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + +def _write_text(path: Path, payload: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(payload, encoding="utf-8") + + +def _content(text: str, role: str = "model") -> Content: + return Content(role=role, parts=[Part.from_text(text=text)]) + + +def _content_text(content: Optional[Content]) -> str: + if not content or not content.parts: + return "" + return "\n".join(part.text or "" for part in content.parts if part.text is not None) + + +def _invocation_user_text(invocation: Invocation) -> str: + return _content_text(invocation.user_content) + + +def _tool_call(name: str, args: dict[str, Any], call_id: str) -> FunctionCall: + return FunctionCall(id=call_id, name=name, args=args) + + +def _tools_to_dicts(tools: list[FunctionCall]) -> list[dict[str, Any]]: + return [{"id": tool.id or "", "name": tool.name or "", "args": dict(tool.args or {})} for tool in tools] + + +def _case_reference_invocation(case: Any) -> Invocation: + if not case.conversation: + raise ValueError(f"eval case {case.eval_id!r} has no reference conversation") + return case.conversation[0] + + +def _case_expected_tools(case: Any) -> list[FunctionCall]: + invocation = _case_reference_invocation(case) + if not invocation.intermediate_data or not isinstance(invocation.intermediate_data, IntermediateData): + return [] + return list(invocation.intermediate_data.tool_uses) + + +def _case_expected_text(case: Any) -> str: + return _content_text(_case_reference_invocation(case).final_response) + + +def _loads_json(text: str) -> bool: + try: + json.loads(text) + return True + except (TypeError, json.JSONDecodeError): + return False + + +def _estimate_tokens(text: str) -> int: + return max(1, len(text.split())) + + +class FakeRubricResponseEvaluator(Evaluator): + """Local deterministic rubric metric used by this example only.""" + + requires_reference = True + + def __init__(self, threshold: Optional[float] = None, eval_metric: Optional[EvalMetric] = None): + if threshold is not None and eval_metric is not None: + raise ValueError("pass either threshold or eval_metric, not both") + self._threshold = threshold if threshold is not None else (eval_metric.threshold if eval_metric else 1.0) + + def evaluate_invocations( + self, + actual_invocations: list[Invocation], + expected_invocations: Optional[list[Invocation]], + ) -> EvaluationResult: + if expected_invocations is None: + raise ValueError("expected_invocations is required for fake_llm_rubric_response") + + per_invocation: list[PerInvocationResult] = [] + for actual, expected in zip(actual_invocations, expected_invocations): + actual_text = _content_text(actual.final_response) + expected_text = _content_text(expected.final_response) + user_text = _invocation_user_text(expected).lower() + exact = actual_text == expected_text + expected_json = _loads_json(expected_text) + actual_json = _loads_json(actual_text) + + rubric_scores = [ + { + "id": "reference_match", + "score": 1.0 if exact else 0.0, + "reason": "final response exactly matches reference" if exact else "final response differs", + }, + { + "id": "json_format", + "score": 1.0 if (not expected_json or actual_json) else 0.0, + "reason": "valid JSON when expected" if (not expected_json or actual_json) else "invalid JSON", + }, + { + "id": "knowledge_grounding", + "score": 0.0 if ("internal" in user_text and not exact) else 1.0, + "reason": "internal knowledge answer is grounded" if exact else "no unsupported internal answer", + }, + ] + score = sum(float(item["score"]) for item in rubric_scores) / len(rubric_scores) + status = EvalStatus.PASSED if score >= self._threshold else EvalStatus.FAILED + failed_ids = [item["id"] for item in rubric_scores if float(item["score"]) < 1.0] + reason = "all fake rubrics passed" if not failed_ids else "failed fake rubrics: " + ", ".join(failed_ids) + per_invocation.append( + PerInvocationResult( + actual_invocation=actual, + expected_invocation=expected, + score=score, + eval_status=status, + reason=reason, + rubric_scores=rubric_scores, + )) + + if not per_invocation: + return EvaluationResult() + overall = sum(float(result.score or 0.0) for result in per_invocation) / len(per_invocation) + return EvaluationResult( + overall_score=overall, + overall_eval_status=EvalStatus.PASSED if overall >= self._threshold else EvalStatus.FAILED, + per_invocation_results=per_invocation, + ) + + +def _register_example_evaluators() -> None: + if "fake_llm_rubric_response" not in EVALUATOR_REGISTRY.list_registered(): + EVALUATOR_REGISTRY.register("fake_llm_rubric_response", FakeRubricResponseEvaluator) + + +class PromptPolicy: + """Tiny prompt interpreter for the deterministic trace-mode agent.""" + + def __init__(self, prompts: dict[str, str]) -> None: + prompt_text = self._positive_prompt_text("\n".join(prompts.values()).lower()) + self.strict_json = self._mentions(prompt_text, ["strict_json_output", "strict json", "json exactly", "compact json"]) + self.strict_tool_args = self._mentions( + prompt_text, + ["strict_tool_arguments", "strict tool", "copy required tool", "copy required tool names and arguments exactly"], + ) + self.training_bias = self._mentions( + prompt_text, + [ + "training_pattern_bias", + "training pattern bias", + "training-set lookup", + "training set lookup", + "training-set lookup shortcut", + "training-vip", + "train-only", + ], + ) + self.no_overfit = self._mentions( + prompt_text, + [ + "do not overfit", + "avoid overfitting", + "do not use training-only", + "do not use train-only", + "preserve validation", + ], + ) + + @staticmethod + def _mentions(text: str, needles: list[str]) -> bool: + return any(needle in text for needle in needles) + + @staticmethod + def _positive_prompt_text(text: str) -> str: + lines = [] + for line in text.splitlines(): + normalized = line.strip().lstrip("-*0123456789. ") + if normalized.startswith(("no ", "missing ", "lacks ", "without ")): + continue + lines.append(line) + return "\n".join(lines) + + +class DeterministicTraceModel: + """Small fake model that creates baseline and overfit candidate traces.""" + + def __init__(self, seed: int) -> None: + self.seed = seed + self.calls = 0 + self.cost = 0.0 + self.token_usage = {"prompt": 0, "completion": 0, "total": 0} + + def predict(self, *, split: str, stage: str, case: Any, prompts: dict[str, str]) -> Invocation: + self.calls += 1 + expected = _case_reference_invocation(case) + expected_text = _content_text(expected.final_response) + expected_tools = _case_expected_tools(case) + query = _invocation_user_text(expected) + text, tools = self._prediction( + split=split, + stage=stage, + case_id=case.eval_id, + query=query, + expected_text=expected_text, + expected_tools=expected_tools, + policy=PromptPolicy(prompts), + ) + self._record_usage(query, text) + return Invocation( + invocation_id=expected.invocation_id, + user_content=deepcopy(expected.user_content), + final_response=_content(text), + intermediate_data=IntermediateData(tool_uses=tools), + creation_timestamp=time.time(), + ) + + def _prediction( + self, + *, + split: str, + stage: str, + case_id: str, + query: str, + expected_text: str, + expected_tools: list[FunctionCall], + policy: PromptPolicy, + ) -> tuple[str, list[FunctionCall]]: + if stage == "baseline": + return self._baseline(split, case_id, query, expected_text, expected_tools) + return self._candidate(split, case_id, query, expected_text, expected_tools, policy=policy) + + def _baseline( + self, + split: str, + case_id: str, + query: str, + expected_text: str, + expected_tools: list[FunctionCall], + ) -> tuple[str, list[FunctionCall]]: + if case_id in {"train_format_json", "val_format_json"}: + return ("The requested total is available, but not returned as JSON.", list(expected_tools)) + if case_id == "train_tool_args": + wrong_tool = _tool_call("get_weather_risk", {"city": "Guangzhou", "date": "2026-07-06"}, "wrong-city") + return (expected_text, [wrong_tool]) + if case_id == "train_knowledge_gap": + return ("I do not have access to the internal launch code.", []) + query_lower = query.lower() + if self._looks_like_knowledge_gap(query_lower): + return ("I do not have access to the required internal knowledge.", []) + if split == "validation" and self._looks_like_regression_case(query_lower): + return (expected_text, list(expected_tools)) + if self._looks_like_format_case(query_lower, expected_text): + return ("The requested value is available, but not returned as JSON.", list(expected_tools)) + if self._looks_like_tool_arg_case(query_lower, expected_tools): + return (expected_text, self._wrong_tool_args(expected_tools, "baseline")) + return (expected_text, list(expected_tools)) + + def _candidate( + self, + split: str, + case_id: str, + query: str, + expected_text: str, + expected_tools: list[FunctionCall], + *, + policy: PromptPolicy, + ) -> tuple[str, list[FunctionCall]]: + fixes_format = policy.strict_json + fixes_tools = policy.strict_tool_args + overfits_validation = policy.training_bias and not policy.no_overfit + baseline_text, baseline_tools = self._baseline(split, case_id, query, expected_text, expected_tools) + + if split == "train" and case_id == "train_format_json" and fixes_format: + return (expected_text, list(expected_tools)) + if split == "train" and case_id == "train_tool_args" and fixes_tools: + return (expected_text, list(expected_tools)) + if case_id == "train_knowledge_gap": + return (baseline_text, baseline_tools) + if case_id == "val_format_json" and fixes_format: + return (expected_text, list(expected_tools)) + if case_id == "val_critical_discount" and overfits_validation: + wrong_tool = _tool_call("lookup_customer_discount", {"customer_id": "C7", "tier": "training-vip"}, + "overfit-discount") + return ("{\"customer_id\":\"C7\",\"discount_percent\":15}", [wrong_tool]) + if case_id == "val_stable_refund" and overfits_validation: + wrong_tool = _tool_call("lookup_refund_sla", {"ticket_id": "R55", "region": "train-only"}, "overfit-refund") + return ("{\"ticket_id\":\"R55\",\"sla_hours\":24}", [wrong_tool]) + query_lower = query.lower() + if self._looks_like_knowledge_gap(query_lower): + return ("I do not have access to the required internal knowledge.", []) + if split == "validation" and self._looks_like_regression_case(query_lower) and overfits_validation: + return (self._overfit_response(expected_text), self._wrong_tool_args(expected_tools, "overfit")) + if split == "train" and ( + (fixes_format and self._looks_like_format_case(query_lower, expected_text)) + or (fixes_tools and self._looks_like_tool_arg_case(query_lower, expected_tools))): + return (expected_text, list(expected_tools)) + if fixes_format and self._looks_like_format_case(query_lower, expected_text): + return (expected_text, list(expected_tools)) + return (baseline_text, baseline_tools) + + def _record_usage(self, prompt: str, completion: str) -> None: + prompt_tokens = _estimate_tokens(prompt) + completion_tokens = _estimate_tokens(completion) + self.token_usage["prompt"] += prompt_tokens + self.token_usage["completion"] += completion_tokens + self.token_usage["total"] += prompt_tokens + completion_tokens + + @staticmethod + def _looks_like_knowledge_gap(query_lower: str) -> bool: + return any(word in query_lower for word in ("internal", "private", "secret", "launch code", "knowledge gap")) + + @staticmethod + def _looks_like_format_case(query_lower: str, expected_text: str) -> bool: + return ("json" in query_lower or _loads_json(expected_text)) and _loads_json(expected_text) + + @staticmethod + def _looks_like_tool_arg_case(query_lower: str, expected_tools: list[FunctionCall]) -> bool: + if not expected_tools: + return False + return any(word in query_lower for word in ("tool", "argument", "args", "weather", "risk", "lookup")) + + @staticmethod + def _looks_like_regression_case(query_lower: str) -> bool: + return any(word in query_lower for word in ("critical", "discount", "vip", "refund", "sla", "stable")) + + @staticmethod + def _wrong_tool_args(expected_tools: list[FunctionCall], marker: str) -> list[FunctionCall]: + if not expected_tools: + return [] + wrong_tools = [] + for index, tool in enumerate(expected_tools): + args = dict(tool.args or {}) + if index == 0: + if args: + first_key = sorted(args)[0] + args[first_key] = f"{args[first_key]}-{marker}" + else: + args["overfit_marker"] = marker + wrong_tools.append(_tool_call(tool.name or "unknown_tool", args, f"{marker}-{index}")) + return wrong_tools + + @staticmethod + def _overfit_response(expected_text: str) -> str: + try: + parsed = json.loads(expected_text) + except json.JSONDecodeError: + return f"{expected_text} (overfit candidate)" + if isinstance(parsed, dict): + for key, value in parsed.items(): + if isinstance(value, bool): + parsed[key] = not value + break + if isinstance(value, (int, float)): + parsed[key] = value + 1 + break + if isinstance(value, str): + parsed[key] = f"{value}-overfit" + break + return json.dumps(parsed, ensure_ascii=False, separators=(",", ":")) + + +def _make_trace_eval_set( + reference: EvalSet, + *, + split: str, + stage: str, + model: DeterministicTraceModel, + prompts: dict[str, str], +) -> EvalSet: + cases = [] + for case in reference.eval_cases: + cases.append( + case.model_copy( + deep=True, + update={ + "eval_mode": "trace", + "actual_conversation": [model.predict(split=split, stage=stage, case=case, prompts=prompts)], + }, + )) + return EvalSet( + eval_set_id=f"{reference.eval_set_id}_{stage}_trace", + app_name=reference.app_name, + name=f"{reference.name or reference.eval_set_id} ({stage} trace)", + description=f"Generated {stage} trace for {split}", + eval_cases=cases, + ) + + +async def _evaluate_trace_set(trace_set: EvalSet, eval_config: EvalConfig) -> dict[str, Any]: + failed_summary, details, result_lines, eval_results = await AgentEvaluator.evaluate_eval_set( + trace_set, + eval_config=eval_config, + print_detailed_results=False, + ) + return { + "failed_summary": failed_summary, + "details": details, + "result_lines": result_lines, + "eval_results_by_eval_id": eval_results, + } + + +def _status_name(status: EvalStatus) -> str: + return status.name.lower() + + +def _metric_summary(metric: Any) -> dict[str, Any]: + return { + "score": metric.score, + "status": _status_name(metric.eval_status), + "threshold": metric.threshold, + "reason": metric.details.reason if metric.details else None, + "rubric_scores": metric.details.rubric_scores if metric.details else None, + } + + +def _case_metrics(result: Any) -> MetricMap: + return {metric.metric_name: _metric_summary(metric) for metric in result.overall_eval_metric_results} + + +def _case_score(metrics: MetricMap) -> float: + if not metrics: + return 0.0 + return sum(float(metric["score"] or 0.0) for metric in metrics.values()) / len(metrics) + + +def _compare_tools(actual: list[dict[str, Any]], expected: list[dict[str, Any]]) -> tuple[bool, bool]: + name_mismatch = len(actual) != len(expected) + args_mismatch = len(actual) != len(expected) + for actual_tool, expected_tool in zip(actual, expected): + if actual_tool["name"] != expected_tool["name"]: + name_mismatch = True + if actual_tool["args"] != expected_tool["args"]: + args_mismatch = True + return name_mismatch, args_mismatch + + +def _attribute_failures(result: Any) -> list[dict[str, Any]]: + metrics = _case_metrics(result) + first_invocation = result.eval_metric_result_per_invocation[0] if result.eval_metric_result_per_invocation else None + if first_invocation is None: + return [{"type": "not_evaluated", "reason": result.error_message or "no invocation was evaluated"}] + + actual_text = _content_text(first_invocation.actual_invocation.final_response) + expected_text = _content_text(first_invocation.expected_invocation.final_response if first_invocation.expected_invocation else None) + query = _invocation_user_text(first_invocation.expected_invocation) if first_invocation.expected_invocation else "" + actual_tools = _tools_to_dicts(first_invocation.actual_invocation.intermediate_data.tool_uses + if isinstance(first_invocation.actual_invocation.intermediate_data, IntermediateData) + else []) + expected_tools = _tools_to_dicts(first_invocation.expected_invocation.intermediate_data.tool_uses + if first_invocation.expected_invocation + and isinstance(first_invocation.expected_invocation.intermediate_data, IntermediateData) + else []) + reasons: list[dict[str, Any]] = [] + + final_metric = metrics.get("final_response_avg_score") + if final_metric and final_metric["status"] == "failed": + if "internal" in query.lower() or "knowledge" in query.lower(): + reasons.append({"type": "knowledge_recall_insufficient", "reason": "answer missed a required internal fact"}) + if _loads_json(expected_text) and not _loads_json(actual_text): + reasons.append({"type": "format_noncompliance", "reason": "expected JSON but actual response is not valid JSON"}) + if not reasons: + reasons.append({"type": "final_response_mismatch", "reason": "actual final response differs from reference"}) + + tool_metric = metrics.get("tool_trajectory_avg_score") + if tool_metric and tool_metric["status"] == "failed": + if not actual_tools and expected_tools: + reasons.append({"type": "tool_call_error", "reason": "expected tool call was missing"}) + else: + name_mismatch, args_mismatch = _compare_tools(actual_tools, expected_tools) + if name_mismatch: + reasons.append({"type": "tool_call_error", "reason": "tool call name or count differs from reference"}) + if args_mismatch: + reasons.append({"type": "tool_argument_error", "reason": "tool arguments differ from reference"}) + + rubric_metric = metrics.get("fake_llm_rubric_response") or metrics.get("llm_rubric_response") + if rubric_metric and rubric_metric["status"] == "failed": + reasons.append({"type": "llm_rubric_not_met", "reason": rubric_metric["reason"] or "rubric score below threshold"}) + + if result.final_eval_status == EvalStatus.FAILED and not reasons: + reasons.append({"type": "unknown_failure", "reason": result.error_message or "case failed without metric reason"}) + return reasons + + +def _summarize_phase(*, split: str, stage: str, trace_set: EvalSet, evaluation: dict[str, Any]) -> dict[str, Any]: + raw_results = evaluation["eval_results_by_eval_id"] + cases = [] + metric_values: dict[str, list[float]] = defaultdict(list) + failures_by_type: Counter[str] = Counter() + pass_count = 0 + + for eval_id in sorted(raw_results): + result = raw_results[eval_id][0] + metrics = _case_metrics(result) + for name, metric in metrics.items(): + metric_values[name].append(float(metric["score"] or 0.0)) + failures = _attribute_failures(result) + for failure in failures: + failures_by_type[failure["type"]] += 1 + if result.final_eval_status == EvalStatus.PASSED: + pass_count += 1 + + first_invocation = result.eval_metric_result_per_invocation[0] if result.eval_metric_result_per_invocation else None + actual_tools: list[dict[str, Any]] = [] + expected_tools: list[dict[str, Any]] = [] + actual_response = "" + expected_response = "" + query = "" + if first_invocation is not None: + actual = first_invocation.actual_invocation + expected = first_invocation.expected_invocation + actual_response = _content_text(actual.final_response) + expected_response = _content_text(expected.final_response if expected else None) + query = _invocation_user_text(expected) if expected else _invocation_user_text(actual) + if isinstance(actual.intermediate_data, IntermediateData): + actual_tools = _tools_to_dicts(actual.intermediate_data.tool_uses) + if expected and isinstance(expected.intermediate_data, IntermediateData): + expected_tools = _tools_to_dicts(expected.intermediate_data.tool_uses) + + cases.append({ + "eval_id": eval_id, + "query": query, + "status": _status_name(result.final_eval_status), + "score": round(_case_score(metrics), 6), + "metrics": metrics, + "actual_response": actual_response, + "expected_response": expected_response, + "actual_tools": actual_tools, + "expected_tools": expected_tools, + "failure_reasons": failures, + "trace": { + "session_id": result.session_id, + "run_id": result.run_id, + }, + }) + + total_cases = len(cases) + metric_breakdown = { + name: round(sum(values) / len(values), 6) if values else 0.0 for name, values in sorted(metric_values.items()) + } + total_score = round(sum(case["score"] for case in cases) / total_cases, 6) if total_cases else 0.0 + return { + "split": split, + "stage": stage, + "eval_set_id": trace_set.eval_set_id, + "case_count": total_cases, + "pass_count": pass_count, + "pass_rate": round(pass_count / total_cases, 6) if total_cases else 0.0, + "total_score": total_score, + "metric_breakdown": metric_breakdown, + "cases": cases, + "failure_attribution": { + "by_type": dict(sorted(failures_by_type.items())), + "failed_case_count": sum(1 for case in cases if case["status"] == "failed"), + }, + } + + +def _case_index(phase: dict[str, Any]) -> dict[str, dict[str, Any]]: + return {case["eval_id"]: case for case in phase["cases"]} + + +def _build_delta(baseline: dict[str, Any], candidate: dict[str, Any]) -> dict[str, Any]: + baseline_cases = _case_index(baseline) + candidate_cases = _case_index(candidate) + case_deltas = [] + new_passes = [] + new_failures = [] + improved = [] + regressed = [] + + for eval_id in sorted(set(baseline_cases) | set(candidate_cases)): + before = baseline_cases.get(eval_id) + after = candidate_cases.get(eval_id) + if before is None or after is None: + continue + score_delta = round(float(after["score"]) - float(before["score"]), 6) + status_delta = f"{before['status']}->{after['status']}" + if before["status"] != "passed" and after["status"] == "passed": + new_passes.append(eval_id) + if before["status"] == "passed" and after["status"] != "passed": + new_failures.append(eval_id) + if score_delta > 0: + improved.append(eval_id) + if score_delta < 0: + regressed.append(eval_id) + case_deltas.append({ + "eval_id": eval_id, + "baseline_score": before["score"], + "candidate_score": after["score"], + "score_delta": score_delta, + "status_delta": status_delta, + "classification": ( + "new_pass" if eval_id in new_passes else + "new_failure" if eval_id in new_failures else + "score_improved" if score_delta > 0 else + "score_regressed" if score_delta < 0 else "unchanged" + ), + }) + + return { + "baseline_total_score": baseline["total_score"], + "candidate_total_score": candidate["total_score"], + "total_score_delta": round(float(candidate["total_score"]) - float(baseline["total_score"]), 6), + "baseline_pass_rate": baseline["pass_rate"], + "candidate_pass_rate": candidate["pass_rate"], + "pass_rate_delta": round(float(candidate["pass_rate"]) - float(baseline["pass_rate"]), 6), + "new_passes": new_passes, + "new_failures": new_failures, + "score_improved": improved, + "score_regressed": regressed, + "case_deltas": case_deltas, + } + + +def _apply_gate(delta: dict[str, Any], gate_config: dict[str, Any], *, total_cost: float) -> dict[str, Any]: + checks = {} + reasons = [] + + min_improvement = float(gate_config.get("min_validation_score_improvement", 0.0)) + checks["validation_score_improvement"] = { + "passed": delta["total_score_delta"] >= min_improvement, + "observed": delta["total_score_delta"], + "required": min_improvement, + } + if not checks["validation_score_improvement"]["passed"]: + reasons.append( + f"validation score delta {delta['total_score_delta']:+.4f} is below required {min_improvement:+.4f}") + + allow_new_hard_failures = bool(gate_config.get("allow_new_hard_failures", False)) + checks["no_new_hard_failures"] = { + "passed": allow_new_hard_failures or not delta["new_failures"], + "new_failures": delta["new_failures"], + "allow_new_hard_failures": allow_new_hard_failures, + } + if not checks["no_new_hard_failures"]["passed"]: + reasons.append("candidate introduced new hard failures: " + ", ".join(delta["new_failures"])) + + critical_ids = list(gate_config.get("critical_case_ids", [])) + critical_regressions = [case_id for case_id in critical_ids if case_id in delta["score_regressed"]] + checks["critical_cases_not_degraded"] = { + "passed": not critical_regressions, + "critical_case_ids": critical_ids, + "regressed": critical_regressions, + } + if critical_regressions: + reasons.append("critical cases regressed: " + ", ".join(critical_regressions)) + + max_cost = float(gate_config.get("max_cost", float("inf"))) + checks["cost_budget"] = { + "passed": total_cost <= max_cost, + "observed": total_cost, + "budget": max_cost, + } + if total_cost > max_cost: + reasons.append(f"run cost {total_cost:.4f} exceeded budget {max_cost:.4f}") + + accepted = all(check["passed"] for check in checks.values()) + if accepted: + reasons.append("candidate passed every configured gate") + return { + "accepted": accepted, + "decision": "accept" if accepted else "reject", + "checks": checks, + "reasons": reasons, + } + + +def _build_target_prompt(config: dict[str, Any], base_dir: Path) -> TargetPrompt: + target_prompt = TargetPrompt() + for item in config["optimize"].get("target_prompts", []): + path = (base_dir / item["path"]).resolve() + target_prompt.add_path(item["name"], str(path)) + return target_prompt + + +def _fake_optimize_prompts( + baseline_prompts: dict[str, str], + config: dict[str, Any], + *, + input_failure_attribution: dict[str, Any], +) -> tuple[dict[str, str], list[dict[str, Any]], dict[str, Any]]: + started = time.monotonic() + patch_lines = list(config["optimize"].get("fake_model", {}).get("candidate_patch", [])) + candidate_prompts = dict(baseline_prompts) + system = candidate_prompts.get("system_prompt", "") + candidate_prompts["system_prompt"] = system.rstrip() + "\n\nOptimization candidate:\n- " + "\n- ".join(patch_lines) + "\n" + round_record = { + "round": 1, + "backend": "fake", + "optimized_field_names": ["system_prompt"], + "input_failure_attribution": input_failure_attribution, + "candidate_prompts": candidate_prompts, + "candidate_prompt_preview": { + "system_prompt": candidate_prompts["system_prompt"][-600:], + }, + "accepted_by_optimizer": True, + "acceptance_reason": "fake optimizer produced one deterministic candidate for downstream gate validation", + "cost": 0.0, + "duration_seconds": round(time.monotonic() - started, 6), + } + return candidate_prompts, [round_record], { + "status": "SUCCEEDED", + "algorithm": "deterministic_fake_patch", + "total_reflection_lm_calls": 0, + "total_token_usage": { + "prompt": 0, + "completion": 0, + "total": 0, + }, + "result_artifacts": {}, + } + + +def _deep_update(base: dict[str, Any], patch: dict[str, Any]) -> dict[str, Any]: + out = deepcopy(base) + for key, value in patch.items(): + if isinstance(value, dict) and isinstance(out.get(key), dict): + out[key] = _deep_update(out[key], value) + else: + out[key] = deepcopy(value) + return out + + +def _env_names_in(value: Any) -> set[str]: + if isinstance(value, dict): + found: set[str] = set() + for child in value.values(): + found.update(_env_names_in(child)) + return found + if isinstance(value, list): + found = set() + for child in value: + found.update(_env_names_in(child)) + return found + if not isinstance(value, str): + return set() + names: set[str] = set() + for match in re.finditer(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}|\$([A-Za-z_][A-Za-z0-9_]*)|%([A-Za-z_][A-Za-z0-9_]*)%", value): + names.add(next(group for group in match.groups() if group)) + return names + + +def _response_only_eval_config(config: dict[str, Any]) -> dict[str, Any]: + metrics = [] + for metric in config["evaluate"].get("metrics", []): + name = metric.get("metric_name") or metric.get("metricName") + if name in {"tool_trajectory_avg_score", "llm_rubric_knowledge_recall"}: + continue + metrics.append(deepcopy(metric)) + if not metrics: + raise ValueError("agent_optimizer backend needs at least one response-only metric") + return { + "metrics": metrics, + "num_runs": int(config["evaluate"].get("num_runs", 1)), + } + + +def _build_agent_optimizer_config(config: dict[str, Any], *, output_dir: Path) -> dict[str, Any]: + optimize_cfg = config.get("optimize", {}) + agent_cfg = optimize_cfg.get("agent_optimizer", {}) + + os.environ.setdefault("TRPC_AGENT_OPT_MODEL", str(agent_cfg.get("model_name", "deepseek-v4-pro"))) + os.environ.setdefault("TRPC_AGENT_OPT_BASE_URL", str(agent_cfg.get("base_url", "https://api.deepseek.com/v1"))) + + default_algorithm = { + "name": "gepa_reflective", + "seed": int(optimize_cfg.get("seed", 0)), + "reflection_lm": { + "provider_name": "openai", + "model_name": "${TRPC_AGENT_OPT_MODEL}", + "base_url": "${TRPC_AGENT_OPT_BASE_URL}", + "api_key": "${TRPC_AGENT_OPT_API_KEY}", + "generation_config": { + "max_tokens": 4096, + "temperature": 0.4, + }, + }, + "candidate_selection_strategy": "pareto", + "module_selector": "round_robin", + "frontier_type": "instance", + "reflection_minibatch_size": 2, + "skip_perfect_score": False, + "use_merge": False, + "max_metric_calls": 12, + "max_candidate_proposals": int(optimize_cfg.get("max_rounds", 1)), + "max_iterations_without_improvement": 2, + } + algorithm = _deep_update(default_algorithm, agent_cfg.get("algorithm", {})) + evaluate = deepcopy(agent_cfg.get("evaluate") or _response_only_eval_config(config)) + metric_names = [metric.get("metric_name") or metric.get("metricName") for metric in evaluate.get("metrics", [])] + return { + "evaluate": evaluate, + "optimize": { + "eval_case_parallelism": int(agent_cfg.get("eval_case_parallelism", 1)), + "stop": { + "required_metrics": agent_cfg.get("required_metrics", [name for name in metric_names if name]), + }, + "algorithm": algorithm, + }, + "_artifact_output_dir": str(output_dir / str(agent_cfg.get("artifact_dir", "agent_optimizer_run"))), + } + + +def _assert_required_optimizer_env(agent_optimizer_config: dict[str, Any]) -> None: + required = _env_names_in(agent_optimizer_config["optimize"]["algorithm"].get("reflection_lm", {})) + missing = sorted(name for name in required if not os.getenv(name)) + if missing: + raise RuntimeError( + "agent_optimizer backend requires environment variable(s): " + + ", ".join(missing) + + ". Do not put API keys in optimizer.json; export them before running.") + + +def _optimizer_round_to_report(record: Any) -> dict[str, Any]: + payload = record.model_dump(mode="json", by_alias=True) + payload["backend"] = "agent_optimizer" + payload["accepted_by_optimizer"] = bool(payload.get("accepted", False)) + payload["cost"] = float(payload.get("round_llm_cost", 0.0) or 0.0) + return payload + + +def _build_optimizer_call_agent( + *, + train_set: EvalSet, + val_set: EvalSet, + target_prompt: TargetPrompt, + model: DeterministicTraceModel, +): + case_by_query: dict[str, tuple[str, Any]] = {} + for split, eval_set in (("train", train_set), ("validation", val_set)): + for case in eval_set.eval_cases: + case_by_query[_invocation_user_text(_case_reference_invocation(case))] = (split, case) + + async def call_agent(query: str) -> str: + split, case = case_by_query[query] + prompts = await target_prompt.read_all() + invocation = model.predict(split=split, stage="candidate", case=case, prompts=prompts) + return _content_text(invocation.final_response) + + return call_agent + + +def _stage_agent_optimizer_evalsets(*, artifact_dir: Path, train_path: Path, val_path: Path) -> tuple[str, str]: + """Copy optimizer input evalsets under the cwd and return colon-safe relative paths.""" + data_dir = artifact_dir / "input_evalsets" + data_dir.mkdir(parents=True, exist_ok=True) + staged_train = data_dir / "train.evalset.json" + staged_val = data_dir / "val.evalset.json" + _write_text(staged_train, train_path.read_text(encoding="utf-8")) + _write_text(staged_val, val_path.read_text(encoding="utf-8")) + return ( + os.path.relpath(staged_train, Path.cwd()).replace("\\", "/"), + os.path.relpath(staged_val, Path.cwd()).replace("\\", "/"), + ) + + +async def _agent_optimizer_prompts( + *, + baseline_prompts: dict[str, str], + config: dict[str, Any], + config_path: Path, + train_path: Path, + val_path: Path, + train_set: EvalSet, + val_set: EvalSet, + output_dir: Path, + target_prompt: TargetPrompt, + model: DeterministicTraceModel, + input_failure_attribution: dict[str, Any], +) -> tuple[dict[str, str], list[dict[str, Any]], dict[str, Any]]: + started = time.monotonic() + runtime_config = _build_agent_optimizer_config(config, output_dir=output_dir) + _assert_required_optimizer_env(runtime_config) + + artifact_dir = Path(runtime_config.pop("_artifact_output_dir")) + staged_train_path, staged_val_path = _stage_agent_optimizer_evalsets( + artifact_dir=artifact_dir, + train_path=train_path, + val_path=val_path, + ) + runtime_config_path = output_dir / "agent_optimizer.config.json" + _write_json(runtime_config_path, runtime_config) + + result = await AgentOptimizer.optimize( + config_path=str(runtime_config_path), + call_agent=_build_optimizer_call_agent( + train_set=train_set, + val_set=val_set, + target_prompt=target_prompt, + model=model, + ), + target_prompt=target_prompt, + train_dataset_path=staged_train_path, + validation_dataset_path=staged_val_path, + output_dir=str(artifact_dir), + update_source=False, + verbose=0, + ) + optimizer_succeeded = result.status == "SUCCEEDED" + candidate_prompts = dict(result.best_prompts if optimizer_succeeded and result.best_prompts else baseline_prompts) + rounds = [_optimizer_round_to_report(record) for record in result.rounds] + if not rounds: + rounds = [{ + "round": 0, + "backend": "agent_optimizer", + "optimized_field_names": [], + "input_failure_attribution": input_failure_attribution, + "candidate_prompts": candidate_prompts, + "accepted_by_optimizer": optimizer_succeeded and candidate_prompts != baseline_prompts, + "acceptance_reason": result.error_message or result.finish_reason, + "cost": float(result.total_llm_cost or 0.0), + "duration_seconds": round(time.monotonic() - started, 6), + }] + summary = _without_secret_fields(result.model_dump(mode="json", by_alias=True)) + summary["candidate_source"] = "best_prompts" if optimizer_succeeded else "baseline_prompts_due_to_optimizer_failure" + summary["input_failure_attribution"] = input_failure_attribution + summary["config_path"] = _display_path(runtime_config_path, config_path.parent) + summary["artifact_dir"] = _display_path(artifact_dir, config_path.parent) + summary["staged_train_evalset"] = staged_train_path + summary["staged_validation_evalset"] = staged_val_path + return candidate_prompts, rounds, summary + + +def _save_trace_sets(output_dir: Path, traces: dict[str, EvalSet], *, base_dir: Path) -> dict[str, str]: + trace_dir = output_dir / "trace_evalsets" + trace_dir.mkdir(parents=True, exist_ok=True) + paths = {} + for name, eval_set in traces.items(): + path = trace_dir / f"{name}.evalset.json" + _write_text(path, eval_set.model_dump_json(indent=2, by_alias=True) + "\n") + paths[name] = _display_path(path, base_dir) + return paths + + +def _render_markdown(report: dict[str, Any]) -> str: + gate = report["gate_decision"] + train_delta = report["delta"]["train"] + delta = report["delta"]["validation"] + lines = [ + "# Evaluation + Optimization Report", + "", + f"- Decision: **{gate['decision'].upper()}**", + f"- Baseline train score: `{train_delta['baseline_total_score']:.4f}`", + f"- Candidate train score: `{train_delta['candidate_total_score']:.4f}`", + f"- Train delta: `{train_delta['total_score_delta']:+.4f}`", + f"- Baseline validation score: `{delta['baseline_total_score']:.4f}`", + f"- Candidate validation score: `{delta['candidate_total_score']:.4f}`", + f"- Validation delta: `{delta['total_score_delta']:+.4f}`", + f"- New passes: `{', '.join(delta['new_passes']) or 'none'}`", + f"- New failures: `{', '.join(delta['new_failures']) or 'none'}`", + "", + "## Gate Reasons", + "", + ] + for reason in gate["reasons"]: + lines.append(f"- {reason}") + + lines.extend([ + "", + "## Train Case Delta", + "", + "| case | baseline | candidate | delta | classification |", + "| --- | ---: | ---: | ---: | --- |", + ]) + for item in train_delta["case_deltas"]: + lines.append( + f"| `{item['eval_id']}` | {item['baseline_score']:.4f} | " + f"{item['candidate_score']:.4f} | {item['score_delta']:+.4f} | {item['classification']} |") + + lines.extend([ + "", + "## Validation Case Delta", + "", + "| case | baseline | candidate | delta | classification |", + "| --- | ---: | ---: | ---: | --- |", + ]) + for item in delta["case_deltas"]: + lines.append( + f"| `{item['eval_id']}` | {item['baseline_score']:.4f} | " + f"{item['candidate_score']:.4f} | {item['score_delta']:+.4f} | {item['classification']} |") + + lines.extend([ + "", + "## Failure Attribution", + "", + ]) + for key, stats in report["failure_attribution"].items(): + by_type = stats["by_type"] + rendered = ", ".join(f"{name}={count}" for name, count in by_type.items()) or "none" + lines.append(f"- `{key}`: {rendered}") + + lines.extend([ + "", + "## Audit", + "", + f"- Seed: `{report['audit']['seed']}`", + f"- Backend: `{report['optimization']['backend']}`", + f"- Duration seconds: `{report['audit']['duration_seconds']:.4f}`", + f"- Total fake/model calls: `{report['optimization']['model_calls']}`", + f"- Total cost: `{report['optimization']['total_cost']:.4f}`", + ]) + return "\n".join(lines) + "\n" + + +async def run_pipeline( + *, + config_path: Path, + train_path: Path, + val_path: Path, + output_dir: Path, + backend_override: Optional[str] = None, +) -> dict[str, Any]: + started_monotonic = time.monotonic() + started_at = _utc_now() + _register_example_evaluators() + + config = _read_json(config_path) + if backend_override: + config.setdefault("optimize", {})["backend"] = backend_override + eval_config = EvalConfig.model_validate(config["evaluate"]) + train_set = EvalSet.model_validate_json(train_path.read_text(encoding="utf-8")) + val_set = EvalSet.model_validate_json(val_path.read_text(encoding="utf-8")) + + seed = int(config.get("optimize", {}).get("seed", 0)) + model = DeterministicTraceModel(seed=seed) + target_prompt = _build_target_prompt(config, config_path.parent) + baseline_prompts = await target_prompt.read_all() + + baseline_trace_sets = { + "baseline_train": _make_trace_eval_set( + train_set, + split="train", + stage="baseline", + model=model, + prompts=baseline_prompts, + ), + "baseline_val": _make_trace_eval_set( + val_set, + split="validation", + stage="baseline", + model=model, + prompts=baseline_prompts, + ), + } + baseline_evaluations = { + name: await _evaluate_trace_set(trace_set, eval_config) for name, trace_set in baseline_trace_sets.items() + } + + phases = { + "baseline_train": _summarize_phase( + split="train", + stage="baseline", + trace_set=baseline_trace_sets["baseline_train"], + evaluation=baseline_evaluations["baseline_train"], + ), + "baseline_validation": _summarize_phase( + split="validation", + stage="baseline", + trace_set=baseline_trace_sets["baseline_val"], + evaluation=baseline_evaluations["baseline_val"], + ), + } + + input_failure_attribution = { + "train": phases["baseline_train"]["failure_attribution"], + "validation": phases["baseline_validation"]["failure_attribution"], + } + backend = config["optimize"].get("backend", "fake") + if backend == "fake": + candidate_prompts, rounds, optimizer_summary = _fake_optimize_prompts( + baseline_prompts, + config, + input_failure_attribution=input_failure_attribution, + ) + elif backend == "agent_optimizer": + candidate_prompts, rounds, optimizer_summary = await _agent_optimizer_prompts( + baseline_prompts=baseline_prompts, + config=config, + config_path=config_path, + train_path=train_path, + val_path=val_path, + train_set=train_set, + val_set=val_set, + output_dir=output_dir, + target_prompt=target_prompt, + model=model, + input_failure_attribution=input_failure_attribution, + ) + else: + raise ValueError(f"unsupported optimize.backend={backend!r}; expected 'fake' or 'agent_optimizer'") + + candidate_trace_sets = { + "candidate_train": _make_trace_eval_set( + train_set, + split="train", + stage="candidate", + model=model, + prompts=candidate_prompts, + ), + "candidate_val": _make_trace_eval_set( + val_set, + split="validation", + stage="candidate", + model=model, + prompts=candidate_prompts, + ), + } + candidate_evaluations = { + name: await _evaluate_trace_set(trace_set, eval_config) for name, trace_set in candidate_trace_sets.items() + } + phases.update({ + "candidate_train": _summarize_phase( + split="train", + stage="candidate", + trace_set=candidate_trace_sets["candidate_train"], + evaluation=candidate_evaluations["candidate_train"], + ), + "candidate_validation": _summarize_phase( + split="validation", + stage="candidate", + trace_set=candidate_trace_sets["candidate_val"], + evaluation=candidate_evaluations["candidate_val"], + ), + }) + trace_sets = {**baseline_trace_sets, **candidate_trace_sets} + + train_delta = _build_delta(phases["baseline_train"], phases["candidate_train"]) + val_delta = _build_delta(phases["baseline_validation"], phases["candidate_validation"]) + optimizer_cost = float(optimizer_summary.get("total_llm_cost", 0.0) or 0.0) + if optimizer_cost <= 0.0: + optimizer_cost = sum(float(round_record.get("cost", 0.0)) for round_record in rounds) + optimizer_usage = optimizer_summary.get("total_token_usage", {}) if isinstance(optimizer_summary, dict) else {} + total_token_usage = { + "prompt": int(model.token_usage.get("prompt", 0)) + int(optimizer_usage.get("prompt", 0) or 0), + "completion": int(model.token_usage.get("completion", 0)) + int(optimizer_usage.get("completion", 0) or 0), + "total": int(model.token_usage.get("total", 0)) + int(optimizer_usage.get("total", 0) or 0), + } + total_cost = model.cost + optimizer_cost + gate_decision = _apply_gate(val_delta, config.get("gate", {}), total_cost=total_cost) + duration = time.monotonic() - started_monotonic + + output_dir.mkdir(parents=True, exist_ok=True) + trace_paths = {} + if config.get("audit", {}).get("save_generated_trace_sets", True): + trace_paths = _save_trace_sets(output_dir, trace_sets, base_dir=config_path.parent) + + report_json_name = config.get("audit", {}).get("report_json", "optimization_report.json") + report_md_name = config.get("audit", {}).get("report_md", "optimization_report.md") + report = { + "schema_version": "eval_optimize_loop.v1", + "experiment": { + "name": config.get("audit", {}).get("experiment_name", "eval_optimize_loop"), + "started_at": started_at, + "finished_at": _utc_now(), + }, + "inputs": { + "train_evalset": _display_path(train_path, config_path.parent), + "validation_evalset": _display_path(val_path, config_path.parent), + "optimizer_config": _display_path(config_path, config_path.parent), + "prompt_sources": [ + { + "name": name, + "path": _display_path(Path(target_prompt.describe_source(name)), config_path.parent), + } for name in target_prompt.names() + ], + }, + "baseline": { + "prompts": baseline_prompts, + "train": phases["baseline_train"], + "validation": phases["baseline_validation"], + }, + "optimization": { + "backend": backend, + "seed": seed, + "optimizer_result": optimizer_summary, + "rounds": rounds, + "candidate_prompts": candidate_prompts, + "model_calls": model.calls, + "optimizer_model_calls": int(optimizer_summary.get("total_reflection_lm_calls", 0) or 0), + "total_cost": total_cost, + "token_usage": total_token_usage, + }, + "candidate": { + "prompts": candidate_prompts, + "train": phases["candidate_train"], + "validation": phases["candidate_validation"], + }, + "delta": { + "train": train_delta, + "validation": val_delta, + }, + "gate_decision": gate_decision, + "failure_attribution": { + "baseline_train": phases["baseline_train"]["failure_attribution"], + "baseline_validation": phases["baseline_validation"]["failure_attribution"], + "candidate_train": phases["candidate_train"]["failure_attribution"], + "candidate_validation": phases["candidate_validation"]["failure_attribution"], + }, + "audit": { + "trace_mode": bool(config.get("audit", {}).get("trace_mode", True)), + "seed": seed, + "repro_config": { + "evaluate": config["evaluate"], + "optimize": _without_secret_fields(config["optimize"]), + "gate": config.get("gate", {}), + }, + "duration_seconds": round(duration, 6), + "artifacts": { + "report_json": _display_path(output_dir / report_json_name, config_path.parent), + "report_md": _display_path(output_dir / report_md_name, config_path.parent), + "trace_evalsets": trace_paths, + }, + }, + } + + _write_json(output_dir / report_json_name, report) + _write_text(output_dir / report_md_name, _render_markdown(report)) + return report + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run the evaluation + optimization closed-loop example.") + parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG) + parser.add_argument("--train", type=Path, default=DEFAULT_TRAIN) + parser.add_argument("--val", type=Path, default=DEFAULT_VAL) + parser.add_argument("--output-dir", type=Path, default=HERE) + parser.add_argument("--backend", choices=["fake", "agent_optimizer"], default=None) + return parser.parse_args() + + +def main() -> None: + args = _parse_args() + report = asyncio.run( + run_pipeline( + config_path=args.config.resolve(), + train_path=args.train.resolve(), + val_path=args.val.resolve(), + output_dir=args.output_dir.resolve(), + backend_override=args.backend, + )) + decision = report["gate_decision"]["decision"] + report_path = report["audit"]["artifacts"]["report_json"] + print(f"Optimization gate decision: {decision}") + print(f"Report written to: {report_path}") + + +if __name__ == "__main__": + main() diff --git a/examples/optimization/eval_optimize_loop/trace_evalsets/baseline_train.evalset.json b/examples/optimization/eval_optimize_loop/trace_evalsets/baseline_train.evalset.json new file mode 100644 index 00000000..fe828e1a --- /dev/null +++ b/examples/optimization/eval_optimize_loop/trace_evalsets/baseline_train.evalset.json @@ -0,0 +1,403 @@ +{ + "evalSetId": "eval_optimize_loop_train_baseline_trace", + "appName": null, + "name": "Evaluation optimization loop train set (baseline trace)", + "description": "Generated baseline trace for train", + "evalCases": [ + { + "evalId": "train_format_json", + "evalMode": "trace", + "conversation": [ + { + "invocationId": "train-format-json", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "Return the order total for order A100 as JSON.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "{\"order_id\":\"A100\",\"total\":128,\"currency\":\"CNY\"}", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "tool-train-format", + "args": { + "order_id": "A100" + }, + "name": "lookup_order_total", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 0.0 + } + ], + "actualConversation": [ + { + "invocationId": "train-format-json", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "Return the order total for order A100 as JSON.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "The requested total is available, but not returned as JSON.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "tool-train-format", + "args": { + "order_id": "A100" + }, + "name": "lookup_order_total", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 1783331066.154771 + } + ], + "conversationScenario": null, + "sessionInput": null, + "contextMessages": null, + "creationTimestamp": 0.0 + }, + { + "evalId": "train_tool_args", + "evalMode": "trace", + "conversation": [ + { + "invocationId": "train-tool-args", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "Check weather risk for Shenzhen on 2026-07-06.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "{\"city\":\"Shenzhen\",\"date\":\"2026-07-06\",\"risk\":\"rain\"}", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "tool-train-weather", + "args": { + "city": "Shenzhen", + "date": "2026-07-06" + }, + "name": "get_weather_risk", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 0.0 + } + ], + "actualConversation": [ + { + "invocationId": "train-tool-args", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "Check weather risk for Shenzhen on 2026-07-06.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "{\"city\":\"Shenzhen\",\"date\":\"2026-07-06\",\"risk\":\"rain\"}", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "wrong-city", + "args": { + "city": "Guangzhou", + "date": "2026-07-06" + }, + "name": "get_weather_risk", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 1783331066.1548839 + } + ], + "conversationScenario": null, + "sessionInput": null, + "contextMessages": null, + "creationTimestamp": 0.0 + }, + { + "evalId": "train_knowledge_gap", + "evalMode": "trace", + "conversation": [ + { + "invocationId": "train-knowledge-gap", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "What is the internal launch code for project nebula?", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "{\"project\":\"nebula\",\"launch_code\":\"NBL-42\"}", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "tool-train-knowledge", + "args": { + "project": "nebula" + }, + "name": "lookup_internal_project", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 0.0 + } + ], + "actualConversation": [ + { + "invocationId": "train-knowledge-gap", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "What is the internal launch code for project nebula?", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "I do not have access to the internal launch code.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 1783331066.154958 + } + ], + "conversationScenario": null, + "sessionInput": null, + "contextMessages": null, + "creationTimestamp": 0.0 + } + ], + "creationTimestamp": 0.0 +} diff --git a/examples/optimization/eval_optimize_loop/trace_evalsets/baseline_val.evalset.json b/examples/optimization/eval_optimize_loop/trace_evalsets/baseline_val.evalset.json new file mode 100644 index 00000000..07337ac2 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/trace_evalsets/baseline_val.evalset.json @@ -0,0 +1,411 @@ +{ + "evalSetId": "eval_optimize_loop_val_baseline_trace", + "appName": null, + "name": "Evaluation optimization loop validation set (baseline trace)", + "description": "Generated baseline trace for validation", + "evalCases": [ + { + "evalId": "val_format_json", + "evalMode": "trace", + "conversation": [ + { + "invocationId": "val-format-json", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "Return invoice T900 tax summary as JSON.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "{\"invoice_id\":\"T900\",\"tax\":35,\"currency\":\"CNY\"}", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "tool-val-invoice", + "args": { + "invoice_id": "T900" + }, + "name": "lookup_invoice_tax", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 0.0 + } + ], + "actualConversation": [ + { + "invocationId": "val-format-json", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "Return invoice T900 tax summary as JSON.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "The requested total is available, but not returned as JSON.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "tool-val-invoice", + "args": { + "invoice_id": "T900" + }, + "name": "lookup_invoice_tax", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 1783331066.1550272 + } + ], + "conversationScenario": null, + "sessionInput": null, + "contextMessages": null, + "creationTimestamp": 0.0 + }, + { + "evalId": "val_critical_discount", + "evalMode": "trace", + "conversation": [ + { + "invocationId": "val-critical-discount", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "Critical case: compute VIP discount for customer C7.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "{\"customer_id\":\"C7\",\"discount_percent\":10}", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "tool-val-discount", + "args": { + "customer_id": "C7" + }, + "name": "lookup_customer_discount", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 0.0 + } + ], + "actualConversation": [ + { + "invocationId": "val-critical-discount", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "Critical case: compute VIP discount for customer C7.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "{\"customer_id\":\"C7\",\"discount_percent\":10}", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "tool-val-discount", + "args": { + "customer_id": "C7" + }, + "name": "lookup_customer_discount", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 1783331066.1551154 + } + ], + "conversationScenario": null, + "sessionInput": null, + "contextMessages": null, + "creationTimestamp": 0.0 + }, + { + "evalId": "val_stable_refund", + "evalMode": "trace", + "conversation": [ + { + "invocationId": "val-stable-refund", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "Check refund SLA for ticket R55.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "{\"ticket_id\":\"R55\",\"sla_hours\":48}", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "tool-val-refund", + "args": { + "ticket_id": "R55" + }, + "name": "lookup_refund_sla", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 0.0 + } + ], + "actualConversation": [ + { + "invocationId": "val-stable-refund", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "Check refund SLA for ticket R55.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "{\"ticket_id\":\"R55\",\"sla_hours\":48}", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "tool-val-refund", + "args": { + "ticket_id": "R55" + }, + "name": "lookup_refund_sla", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 1783331066.1551723 + } + ], + "conversationScenario": null, + "sessionInput": null, + "contextMessages": null, + "creationTimestamp": 0.0 + } + ], + "creationTimestamp": 0.0 +} diff --git a/examples/optimization/eval_optimize_loop/trace_evalsets/candidate_train.evalset.json b/examples/optimization/eval_optimize_loop/trace_evalsets/candidate_train.evalset.json new file mode 100644 index 00000000..81ee628e --- /dev/null +++ b/examples/optimization/eval_optimize_loop/trace_evalsets/candidate_train.evalset.json @@ -0,0 +1,403 @@ +{ + "evalSetId": "eval_optimize_loop_train_candidate_trace", + "appName": null, + "name": "Evaluation optimization loop train set (candidate trace)", + "description": "Generated candidate trace for train", + "evalCases": [ + { + "evalId": "train_format_json", + "evalMode": "trace", + "conversation": [ + { + "invocationId": "train-format-json", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "Return the order total for order A100 as JSON.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "{\"order_id\":\"A100\",\"total\":128,\"currency\":\"CNY\"}", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "tool-train-format", + "args": { + "order_id": "A100" + }, + "name": "lookup_order_total", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 0.0 + } + ], + "actualConversation": [ + { + "invocationId": "train-format-json", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "Return the order total for order A100 as JSON.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "{\"order_id\":\"A100\",\"total\":128,\"currency\":\"CNY\"}", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "tool-train-format", + "args": { + "order_id": "A100" + }, + "name": "lookup_order_total", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 1783331066.1571672 + } + ], + "conversationScenario": null, + "sessionInput": null, + "contextMessages": null, + "creationTimestamp": 0.0 + }, + { + "evalId": "train_tool_args", + "evalMode": "trace", + "conversation": [ + { + "invocationId": "train-tool-args", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "Check weather risk for Shenzhen on 2026-07-06.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "{\"city\":\"Shenzhen\",\"date\":\"2026-07-06\",\"risk\":\"rain\"}", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "tool-train-weather", + "args": { + "city": "Shenzhen", + "date": "2026-07-06" + }, + "name": "get_weather_risk", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 0.0 + } + ], + "actualConversation": [ + { + "invocationId": "train-tool-args", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "Check weather risk for Shenzhen on 2026-07-06.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "{\"city\":\"Shenzhen\",\"date\":\"2026-07-06\",\"risk\":\"rain\"}", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "tool-train-weather", + "args": { + "city": "Shenzhen", + "date": "2026-07-06" + }, + "name": "get_weather_risk", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 1783331066.1572425 + } + ], + "conversationScenario": null, + "sessionInput": null, + "contextMessages": null, + "creationTimestamp": 0.0 + }, + { + "evalId": "train_knowledge_gap", + "evalMode": "trace", + "conversation": [ + { + "invocationId": "train-knowledge-gap", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "What is the internal launch code for project nebula?", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "{\"project\":\"nebula\",\"launch_code\":\"NBL-42\"}", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "tool-train-knowledge", + "args": { + "project": "nebula" + }, + "name": "lookup_internal_project", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 0.0 + } + ], + "actualConversation": [ + { + "invocationId": "train-knowledge-gap", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "What is the internal launch code for project nebula?", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "I do not have access to the internal launch code.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 1783331066.1573014 + } + ], + "conversationScenario": null, + "sessionInput": null, + "contextMessages": null, + "creationTimestamp": 0.0 + } + ], + "creationTimestamp": 0.0 +} diff --git a/examples/optimization/eval_optimize_loop/trace_evalsets/candidate_val.evalset.json b/examples/optimization/eval_optimize_loop/trace_evalsets/candidate_val.evalset.json new file mode 100644 index 00000000..b8177017 --- /dev/null +++ b/examples/optimization/eval_optimize_loop/trace_evalsets/candidate_val.evalset.json @@ -0,0 +1,413 @@ +{ + "evalSetId": "eval_optimize_loop_val_candidate_trace", + "appName": null, + "name": "Evaluation optimization loop validation set (candidate trace)", + "description": "Generated candidate trace for validation", + "evalCases": [ + { + "evalId": "val_format_json", + "evalMode": "trace", + "conversation": [ + { + "invocationId": "val-format-json", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "Return invoice T900 tax summary as JSON.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "{\"invoice_id\":\"T900\",\"tax\":35,\"currency\":\"CNY\"}", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "tool-val-invoice", + "args": { + "invoice_id": "T900" + }, + "name": "lookup_invoice_tax", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 0.0 + } + ], + "actualConversation": [ + { + "invocationId": "val-format-json", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "Return invoice T900 tax summary as JSON.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "{\"invoice_id\":\"T900\",\"tax\":35,\"currency\":\"CNY\"}", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "tool-val-invoice", + "args": { + "invoice_id": "T900" + }, + "name": "lookup_invoice_tax", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 1783331066.1576939 + } + ], + "conversationScenario": null, + "sessionInput": null, + "contextMessages": null, + "creationTimestamp": 0.0 + }, + { + "evalId": "val_critical_discount", + "evalMode": "trace", + "conversation": [ + { + "invocationId": "val-critical-discount", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "Critical case: compute VIP discount for customer C7.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "{\"customer_id\":\"C7\",\"discount_percent\":10}", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "tool-val-discount", + "args": { + "customer_id": "C7" + }, + "name": "lookup_customer_discount", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 0.0 + } + ], + "actualConversation": [ + { + "invocationId": "val-critical-discount", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "Critical case: compute VIP discount for customer C7.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "{\"customer_id\":\"C7\",\"discount_percent\":15}", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "overfit-discount", + "args": { + "customer_id": "C7", + "tier": "training-vip" + }, + "name": "lookup_customer_discount", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 1783331066.1577723 + } + ], + "conversationScenario": null, + "sessionInput": null, + "contextMessages": null, + "creationTimestamp": 0.0 + }, + { + "evalId": "val_stable_refund", + "evalMode": "trace", + "conversation": [ + { + "invocationId": "val-stable-refund", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "Check refund SLA for ticket R55.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "{\"ticket_id\":\"R55\",\"sla_hours\":48}", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "tool-val-refund", + "args": { + "ticket_id": "R55" + }, + "name": "lookup_refund_sla", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 0.0 + } + ], + "actualConversation": [ + { + "invocationId": "val-stable-refund", + "userContent": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "Check refund SLA for ticket R55.", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "user" + }, + "finalResponse": { + "parts": [ + { + "mediaResolution": null, + "codeExecutionResult": null, + "executableCode": null, + "fileData": null, + "functionCall": null, + "functionResponse": null, + "inlineData": null, + "text": "{\"ticket_id\":\"R55\",\"sla_hours\":24}", + "thought": null, + "thoughtSignature": null, + "videoMetadata": null, + "toolCall": null, + "toolResponse": null, + "partMetadata": null + } + ], + "role": "model" + }, + "intermediateData": { + "toolUses": [ + { + "id": "overfit-refund", + "args": { + "ticket_id": "R55", + "region": "train-only" + }, + "name": "lookup_refund_sla", + "partialArgs": null, + "willContinue": null + } + ], + "toolResponses": [], + "intermediateResponses": [] + }, + "creationTimestamp": 1783331066.1578372 + } + ], + "conversationScenario": null, + "sessionInput": null, + "contextMessages": null, + "creationTimestamp": 0.0 + } + ], + "creationTimestamp": 0.0 +} diff --git a/examples/optimization/eval_optimize_loop/train.evalset.json b/examples/optimization/eval_optimize_loop/train.evalset.json new file mode 100644 index 00000000..66b7787c --- /dev/null +++ b/examples/optimization/eval_optimize_loop/train.evalset.json @@ -0,0 +1,113 @@ +{ + "eval_set_id": "eval_optimize_loop_train", + "name": "Evaluation optimization loop train set", + "description": "Three deterministic training cases: optimizable format failure, optimizable tool-argument failure, and an intentionally unfixable knowledge recall failure.", + "eval_cases": [ + { + "eval_id": "train_format_json", + "conversation": [ + { + "invocation_id": "train-format-json", + "user_content": { + "role": "user", + "parts": [ + { + "text": "Return the order total for order A100 as JSON." + } + ] + }, + "final_response": { + "role": "model", + "parts": [ + { + "text": "{\"order_id\":\"A100\",\"total\":128,\"currency\":\"CNY\"}" + } + ] + }, + "intermediate_data": { + "tool_uses": [ + { + "id": "tool-train-format", + "name": "lookup_order_total", + "args": { + "order_id": "A100" + } + } + ] + } + } + ] + }, + { + "eval_id": "train_tool_args", + "conversation": [ + { + "invocation_id": "train-tool-args", + "user_content": { + "role": "user", + "parts": [ + { + "text": "Check weather risk for Shenzhen on 2026-07-06." + } + ] + }, + "final_response": { + "role": "model", + "parts": [ + { + "text": "{\"city\":\"Shenzhen\",\"date\":\"2026-07-06\",\"risk\":\"rain\"}" + } + ] + }, + "intermediate_data": { + "tool_uses": [ + { + "id": "tool-train-weather", + "name": "get_weather_risk", + "args": { + "city": "Shenzhen", + "date": "2026-07-06" + } + } + ] + } + } + ] + }, + { + "eval_id": "train_knowledge_gap", + "conversation": [ + { + "invocation_id": "train-knowledge-gap", + "user_content": { + "role": "user", + "parts": [ + { + "text": "What is the internal launch code for project nebula?" + } + ] + }, + "final_response": { + "role": "model", + "parts": [ + { + "text": "{\"project\":\"nebula\",\"launch_code\":\"NBL-42\"}" + } + ] + }, + "intermediate_data": { + "tool_uses": [ + { + "id": "tool-train-knowledge", + "name": "lookup_internal_project", + "args": { + "project": "nebula" + } + } + ] + } + } + ] + } + ] +} diff --git a/examples/optimization/eval_optimize_loop/val.evalset.json b/examples/optimization/eval_optimize_loop/val.evalset.json new file mode 100644 index 00000000..4a7c1a6b --- /dev/null +++ b/examples/optimization/eval_optimize_loop/val.evalset.json @@ -0,0 +1,112 @@ +{ + "eval_set_id": "eval_optimize_loop_val", + "name": "Evaluation optimization loop validation set", + "description": "Three deterministic validation cases: one improves, two regress after an overfit candidate so the gate must reject it.", + "eval_cases": [ + { + "eval_id": "val_format_json", + "conversation": [ + { + "invocation_id": "val-format-json", + "user_content": { + "role": "user", + "parts": [ + { + "text": "Return invoice T900 tax summary as JSON." + } + ] + }, + "final_response": { + "role": "model", + "parts": [ + { + "text": "{\"invoice_id\":\"T900\",\"tax\":35,\"currency\":\"CNY\"}" + } + ] + }, + "intermediate_data": { + "tool_uses": [ + { + "id": "tool-val-invoice", + "name": "lookup_invoice_tax", + "args": { + "invoice_id": "T900" + } + } + ] + } + } + ] + }, + { + "eval_id": "val_critical_discount", + "conversation": [ + { + "invocation_id": "val-critical-discount", + "user_content": { + "role": "user", + "parts": [ + { + "text": "Critical case: compute VIP discount for customer C7." + } + ] + }, + "final_response": { + "role": "model", + "parts": [ + { + "text": "{\"customer_id\":\"C7\",\"discount_percent\":10}" + } + ] + }, + "intermediate_data": { + "tool_uses": [ + { + "id": "tool-val-discount", + "name": "lookup_customer_discount", + "args": { + "customer_id": "C7" + } + } + ] + } + } + ] + }, + { + "eval_id": "val_stable_refund", + "conversation": [ + { + "invocation_id": "val-stable-refund", + "user_content": { + "role": "user", + "parts": [ + { + "text": "Check refund SLA for ticket R55." + } + ] + }, + "final_response": { + "role": "model", + "parts": [ + { + "text": "{\"ticket_id\":\"R55\",\"sla_hours\":48}" + } + ] + }, + "intermediate_data": { + "tool_uses": [ + { + "id": "tool-val-refund", + "name": "lookup_refund_sla", + "args": { + "ticket_id": "R55" + } + } + ] + } + } + ] + } + ] +} diff --git a/tests/examples/test_eval_optimize_loop.py b/tests/examples/test_eval_optimize_loop.py new file mode 100644 index 00000000..39f96386 --- /dev/null +++ b/tests/examples/test_eval_optimize_loop.py @@ -0,0 +1,221 @@ +# Tencent is pleased to support the open source community by making tRPC-Agent-Python available. +# +# Copyright (C) 2026 Tencent. All rights reserved. +# +# tRPC-Agent-Python is licensed under Apache-2.0. +"""Regression tests for the eval_optimize_loop example.""" + +from __future__ import annotations + +from copy import deepcopy +import json +from pathlib import Path + +import pytest + +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_CONFIG +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_TRAIN +from examples.optimization.eval_optimize_loop.run_pipeline import DEFAULT_VAL +from examples.optimization.eval_optimize_loop.run_pipeline import run_pipeline + + +def _load_json(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def _write_json(path: Path, payload: dict) -> Path: + path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return path + + +def _config_for_tmp(tmp_path: Path, *, critical_case_ids: list[str] | None = None) -> Path: + config = deepcopy(_load_json(DEFAULT_CONFIG)) + prompt_dir = DEFAULT_CONFIG.parent / "prompts" + config["optimize"]["target_prompts"] = [ + {"name": "system_prompt", "path": str(prompt_dir / "system.md")}, + {"name": "skill_prompt", "path": str(prompt_dir / "skill.md")}, + ] + if critical_case_ids is not None: + config["gate"]["critical_case_ids"] = critical_case_ids + return _write_json(tmp_path / "optimizer.json", config) + + +def _prompt_dir(tmp_path: Path, *, system: str | None = None, skill: str | None = None) -> Path: + prompt_dir = tmp_path / "prompts" + prompt_dir.mkdir() + (prompt_dir / "system.md").write_text(system or "Baseline prompt.\n", encoding="utf-8") + (prompt_dir / "skill.md").write_text(skill or "Use tools when needed.\n", encoding="utf-8") + return prompt_dir + + +def _config_with_prompts(tmp_path: Path, prompt_dir: Path) -> Path: + config = deepcopy(_load_json(DEFAULT_CONFIG)) + config["optimize"]["target_prompts"] = [ + {"name": "system_prompt", "path": str(prompt_dir / "system.md")}, + {"name": "skill_prompt", "path": str(prompt_dir / "skill.md")}, + ] + return _write_json(tmp_path / "optimizer.json", config) + + +def _renamed_evalset(source: Path, replacements: dict[str, str]) -> dict: + payload = deepcopy(_load_json(source)) + payload["eval_set_id"] = payload["eval_set_id"] + "_renamed" + for case in payload["eval_cases"]: + case["eval_id"] = replacements.get(case["eval_id"], case["eval_id"]) + return payload + + +@pytest.mark.asyncio +async def test_eval_optimize_loop_rejects_validation_regression(tmp_path: Path): + report = await run_pipeline( + config_path=DEFAULT_CONFIG, + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + output_dir=tmp_path, + ) + + assert report["gate_decision"]["decision"] == "reject" + assert report["delta"]["train"]["total_score_delta"] > 0 + assert report["delta"]["validation"]["total_score_delta"] < 0 + assert report["delta"]["validation"]["new_passes"] == ["val_format_json"] + assert set(report["delta"]["validation"]["new_failures"]) == { + "val_critical_discount", + "val_stable_refund", + } + assert "val_critical_discount" in report["gate_decision"]["checks"]["critical_cases_not_degraded"]["regressed"] + + all_case_ids = {case["eval_id"] for case in report["baseline"]["train"]["cases"]} + all_case_ids.update(case["eval_id"] for case in report["baseline"]["validation"]["cases"]) + assert all_case_ids == { + "train_format_json", + "train_tool_args", + "train_knowledge_gap", + "val_format_json", + "val_critical_discount", + "val_stable_refund", + } + + for phase in ( + report["baseline"]["train"], + report["baseline"]["validation"], + report["candidate"]["train"], + report["candidate"]["validation"], + ): + for case in phase["cases"]: + if case["status"] == "failed": + assert case["failure_reasons"] + + report_json = tmp_path / "optimization_report.json" + report_md = tmp_path / "optimization_report.md" + assert report_json.exists() + assert report_md.exists() + persisted = json.loads(report_json.read_text(encoding="utf-8")) + assert persisted["gate_decision"]["decision"] == "reject" + rendered = report_md.read_text(encoding="utf-8") + assert "Train Case Delta" in rendered + assert "Validation Case Delta" in rendered + assert "knowledge_recall_insufficient" in rendered + + +@pytest.mark.asyncio +async def test_eval_optimize_loop_rejects_semantic_regression_without_public_ids(tmp_path: Path): + train = _renamed_evalset( + DEFAULT_TRAIN, + { + "train_format_json": "hidden_train_json", + "train_tool_args": "hidden_train_weather", + "train_knowledge_gap": "hidden_train_private_fact", + }, + ) + val = _renamed_evalset( + DEFAULT_VAL, + { + "val_format_json": "hidden_val_json", + "val_critical_discount": "hidden_val_critical_vip", + "val_stable_refund": "hidden_val_refund_sla", + }, + ) + train_path = _write_json(tmp_path / "train.evalset.json", train) + val_path = _write_json(tmp_path / "val.evalset.json", val) + config_path = _config_for_tmp(tmp_path, critical_case_ids=["hidden_val_critical_vip"]) + + report = await run_pipeline( + config_path=config_path, + train_path=train_path, + val_path=val_path, + output_dir=tmp_path, + ) + + assert report["gate_decision"]["decision"] == "reject" + assert report["delta"]["train"]["total_score_delta"] > 0 + assert report["delta"]["validation"]["total_score_delta"] < 0 + assert report["delta"]["validation"]["new_passes"] == ["hidden_val_json"] + assert set(report["delta"]["validation"]["new_failures"]) == { + "hidden_val_critical_vip", + "hidden_val_refund_sla", + } + assert "hidden_val_critical_vip" in report["gate_decision"]["checks"]["critical_cases_not_degraded"]["regressed"] + + +@pytest.mark.asyncio +async def test_eval_optimize_loop_accepts_clean_validation_gain(tmp_path: Path): + train_path = DEFAULT_TRAIN + val = deepcopy(_load_json(DEFAULT_VAL)) + val["eval_set_id"] = "eval_optimize_loop_val_gain_only" + val["eval_cases"] = [case for case in val["eval_cases"] if case["eval_id"] == "val_format_json"] + val["eval_cases"][0]["eval_id"] = "hidden_val_json_gain" + val_path = _write_json(tmp_path / "val.evalset.json", val) + config_path = _config_for_tmp(tmp_path, critical_case_ids=[]) + + report = await run_pipeline( + config_path=config_path, + train_path=train_path, + val_path=val_path, + output_dir=tmp_path, + ) + + assert report["gate_decision"]["decision"] == "accept" + assert report["delta"]["validation"]["total_score_delta"] > 0 + assert report["delta"]["validation"]["new_passes"] == ["hidden_val_json_gain"] + assert report["delta"]["validation"]["new_failures"] == [] + assert report["gate_decision"]["reasons"] == ["candidate passed every configured gate"] + + +@pytest.mark.asyncio +async def test_eval_optimize_loop_candidate_behavior_depends_on_candidate_prompt(tmp_path: Path): + config = deepcopy(_load_json(DEFAULT_CONFIG)) + prompt_dir = DEFAULT_CONFIG.parent / "prompts" + config["optimize"]["target_prompts"] = [ + {"name": "system_prompt", "path": str(prompt_dir / "system.md")}, + {"name": "skill_prompt", "path": str(prompt_dir / "skill.md")}, + ] + config["optimize"]["fake_model"]["candidate_patch"] = [] + config_path = _write_json(tmp_path / "optimizer.json", config) + + report = await run_pipeline( + config_path=config_path, + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + output_dir=tmp_path, + ) + + assert report["delta"]["train"]["new_passes"] == [] + assert report["delta"]["validation"]["new_passes"] == [] + assert report["delta"]["validation"]["new_failures"] == [] + assert report["delta"]["validation"]["total_score_delta"] == 0 + + +@pytest.mark.asyncio +async def test_eval_optimize_loop_agent_optimizer_requires_api_key(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + prompt_dir = _prompt_dir(tmp_path) + config_path = _config_with_prompts(tmp_path, prompt_dir) + monkeypatch.delenv("TRPC_AGENT_OPT_API_KEY", raising=False) + + with pytest.raises(RuntimeError, match="TRPC_AGENT_OPT_API_KEY"): + await run_pipeline( + config_path=config_path, + train_path=DEFAULT_TRAIN, + val_path=DEFAULT_VAL, + output_dir=tmp_path, + backend_override="agent_optimizer", + ) diff --git a/trpc_agent_sdk/evaluation/_optimize_gepa_adapter.py b/trpc_agent_sdk/evaluation/_optimize_gepa_adapter.py index 34fc4dd2..bae3e264 100644 --- a/trpc_agent_sdk/evaluation/_optimize_gepa_adapter.py +++ b/trpc_agent_sdk/evaluation/_optimize_gepa_adapter.py @@ -25,6 +25,7 @@ from __future__ import annotations import asyncio +import os import tempfile import uuid from pathlib import Path @@ -532,6 +533,7 @@ def __init__( num_runs: int = 1, case_parallelism: Optional[int] = None, top_k_per_case: int = 2, + output_dir: Optional[str] = None, ) -> None: self.target_prompt = target_prompt self.eval_config = eval_config @@ -543,6 +545,7 @@ def __init__( self.callbacks = callbacks self.num_runs = num_runs self.case_parallelism = case_parallelism + self.output_dir = output_dir self._top_k = max(0, int(top_k_per_case)) self._best_history: dict[str, list[dict[str, Any]]] = {} from ._optimize_evaluator_call import EvaluationOutcome # local to avoid cycle @@ -608,7 +611,12 @@ def evaluate( loop = self._get_or_create_loop() loop.run_until_complete(self.target_prompt.write_all(candidate)) - with tempfile.TemporaryDirectory() as tmp: + if getattr(self, "output_dir", None): + tmp_parent = Path(self.output_dir) / "tmp_batches" + else: + tmp_parent = Path.cwd() / "tmp" / "optimizer_batches" + tmp_parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=tmp_parent) as tmp: tmp_path = Path(tmp) evalset_path = tmp_path / "batch.evalset.json" metrics_path = tmp_path / "batch.metrics.json" @@ -638,8 +646,14 @@ def evaluate( outcome = loop.run_until_complete( run_evaluator( - eval_dataset_path=str(evalset_path), - eval_metrics_path=str(metrics_path), + eval_dataset_path=os.path.relpath( + evalset_path, + Path.cwd(), + ).replace("\\", "/"), + eval_metrics_path=os.path.relpath( + metrics_path, + Path.cwd(), + ).replace("\\", "/"), call_agent=self.call_agent, callbacks=self.callbacks, num_runs=self.num_runs, diff --git a/trpc_agent_sdk/evaluation/_optimize_gepa_reflective.py b/trpc_agent_sdk/evaluation/_optimize_gepa_reflective.py index 322340eb..d74921a8 100644 --- a/trpc_agent_sdk/evaluation/_optimize_gepa_reflective.py +++ b/trpc_agent_sdk/evaluation/_optimize_gepa_reflective.py @@ -462,6 +462,7 @@ async def run( num_runs=self.config.evaluate.num_runs, case_parallelism=self.config.optimize.eval_case_parallelism, top_k_per_case=int(algo.reflection_history_top_k), + output_dir=self.output_dir, ) reflection_lm = _OptimizeModelCallable(algo.reflection_lm)