From 94d825283659c017d0b6d5d94c706c58cfd48374 Mon Sep 17 00:00:00 2001 From: TillQuandel Date: Mon, 13 Jul 2026 20:13:36 +0200 Subject: [PATCH] feat(db): pipeline_runs.profile + run_start-Trace-Profil persistieren (#235) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Das aktive Runtime-Profil (legacy/fast/balanced/quality) stand bisher nur im Stdout-Log — pipeline_runs hatte keine Profil-Spalte, das Trace run_start-Event nur das Agent->Modell-Mapping. Zwei Läufe mit unterschiedlichem Profil waren in DB/Trace nachträglich nicht unterscheidbar (Voraussetzung für die A/B-Messrunde 2). Additiv nach dem n_extracted-Muster (PR #220): - shared/db_schema.py: Spalte profile TEXT DEFAULT '' in pipeline_runs - generative/db.py: Migration via _add_column (Bestands-DBs) + insert_run schreibt profile (Default "") - generative/orchestrator.py: runtime_config.profile ins insert_run-Dict, Profil an trace_run_start durchgereicht - generative/agents/tracing.py: trace_run_start(model_config, profile="") Neue Tests (RED vor dem Fix, GREEN danach): Roundtrip-Persistenz + PRAGMA-Spalten-Check + Default-Verhalten in test_db.py, run_start-Trace-Profil in test_per_agent_tracking.py. --- generative/agents/tracing.py | 5 +- generative/db.py | 10 ++-- generative/orchestrator.py | 3 +- generative/tests/test_db.py | 58 +++++++++++++++++++++ generative/tests/test_per_agent_tracking.py | 16 ++++++ shared/db_schema.py | 3 +- 6 files changed, 88 insertions(+), 7 deletions(-) diff --git a/generative/agents/tracing.py b/generative/agents/tracing.py index 919576b..a47cb4d 100644 --- a/generative/agents/tracing.py +++ b/generative/agents/tracing.py @@ -73,14 +73,15 @@ def trace_event(agent: str, event_type: str, data: dict) -> None: ) -def trace_run_start(model_config: dict) -> None: - """Schreibt Run-Start-Entry mit Model-Config.""" +def trace_run_start(model_config: dict, profile: str = "") -> None: + """Schreibt Run-Start-Entry mit Model-Config + aktivem Runtime-Profil (#235).""" _backend.write( { "ts": time.strftime("%Y-%m-%dT%H:%M:%S"), "type": "run_start", "run_id": _RUN_ID, "model_config": model_config, + "profile": profile, } ) diff --git a/generative/db.py b/generative/db.py index 9e791a4..725f9c8 100644 --- a/generative/db.py +++ b/generative/db.py @@ -64,6 +64,9 @@ def init_db(path: Path = DB_PATH) -> None: _add_column(conn, "pipeline_runs", "n_words INT DEFAULT 0") _add_column(conn, "pipeline_runs", "model TEXT DEFAULT ''") _add_column(conn, "pipeline_runs", "cost_usd REAL DEFAULT 0.0") + # #235: aktives Runtime-Profil (legacy/fast/balanced/quality) mitschreiben — + # bisher nur im Stdout-Log, dadurch Alt- und A/B-Läufe in der DB ununterscheidbar. + _add_column(conn, "pipeline_runs", "profile TEXT DEFAULT ''") _add_column(conn, "note_evals", "anchor_rate REAL") # Anker-Roh-Counts: für die gepoolte Halluzinationsrate (Σ halluziniert / # Σ gesamt) im Dashboard — die Pipeline berechnet sie ohnehin, persistiert @@ -102,7 +105,7 @@ def insert_run(conn: sqlite3.Connection, data: dict) -> None: run_id, timestamp, pipeline_version, pdf_source, pdf_key, pdf_label, n_generated, n_extracted, n_vault, n_inbox, n_merge, n_dropped, n_words, model, tokens_total, tokens_input, tokens_output, tokens_cache_read, - duration_s, eval_version + duration_s, eval_version, profile n_generated = geschriebene Notes (historische Semantik, unangetastet). n_extracted = "nach Planner/Extractor generiert" (Funnel-Top, #197). Fehlt @@ -115,12 +118,12 @@ def insert_run(conn: sqlite3.Connection, data: dict) -> None: (run_id, timestamp, pipeline_version, pdf_source, pdf_key, pdf_label, n_generated, n_extracted, n_vault, n_inbox, n_merge, n_dropped, n_words, model, cost_usd, tokens_total, tokens_input, tokens_output, tokens_cache_read, - duration_s, eval_version, fully_cached) + duration_s, eval_version, fully_cached, profile) VALUES (:run_id, :timestamp, :pipeline_version, :pdf_source, :pdf_key, :pdf_label, :n_generated, :n_extracted, :n_vault, :n_inbox, :n_merge, :n_dropped, :n_words, :model, :cost_usd, :tokens_total, :tokens_input, :tokens_output, :tokens_cache_read, - :duration_s, :eval_version, :fully_cached) + :duration_s, :eval_version, :fully_cached, :profile) """, { "run_id": data.get("run_id"), @@ -145,6 +148,7 @@ def insert_run(conn: sqlite3.Connection, data: dict) -> None: "duration_s": data.get("duration_s", 0.0), "eval_version": data.get("eval_version"), "fully_cached": 1 if (data.get("tokens_total", 0) == 0 and data.get("duration_s", 0) > 0) else 0, + "profile": data.get("profile", ""), }, ) diff --git a/generative/orchestrator.py b/generative/orchestrator.py index a9f1d1f..64e9382 100644 --- a/generative/orchestrator.py +++ b/generative/orchestrator.py @@ -1917,7 +1917,7 @@ def _run_extraction_stages( from generative.agents.base import trace_run_start as _trace_run_start from generative.config import MODEL_CONFIG as _MODEL_CONFIG - _trace_run_start(_MODEL_CONFIG) + _trace_run_start(_MODEL_CONFIG, profile=getattr(runtime_config, "profile", "")) # --- Schritt 1: PDF → Text + Metadata → Chunks --- print("[1/7] PDF extrahieren und chunken…") @@ -2784,6 +2784,7 @@ def main(argv: list[str] | None = None): "tokens_output": _tok_out, "tokens_cache_read": _tok_cache_r, "duration_s": _wall_s, + "profile": runtime_config.profile, }, ) except Exception as _db_err: diff --git a/generative/tests/test_db.py b/generative/tests/test_db.py index bfeeb55..b0078dd 100644 --- a/generative/tests/test_db.py +++ b/generative/tests/test_db.py @@ -45,3 +45,61 @@ def test_insert_run_stores_cost_usd(): assert abs(row[0] - 0.1234) < 0.0001 finally: os.unlink(path) + + +# --- #235: Laufzeit-Profil persistieren ------------------------------------ + + +def test_pipeline_runs_has_profile_column(tmp_path): + from generative import db + + path = tmp_path / "test.db" + db.init_db(path) + with db.get_db(path) as conn: + cols = [r[1] for r in conn.execute("PRAGMA table_info(pipeline_runs)").fetchall()] + assert "profile" in cols + + +def test_insert_run_stores_profile(tmp_path): + from generative import db + + path = tmp_path / "test.db" + db.init_db(path) + with db.get_db(path) as conn: + db.insert_run( + conn, + { + "run_id": "test-run-profile", + "pipeline_version": "v0.0.1", + "profile": "balanced", + }, + ) + conn2 = sqlite3.connect(str(path)) + try: + row = conn2.execute("SELECT profile FROM pipeline_runs WHERE run_id='test-run-profile'").fetchone() + finally: + conn2.close() + assert row is not None + assert row[0] == "balanced" + + +def test_insert_run_profile_defaults_to_empty_string(tmp_path): + from generative import db + + path = tmp_path / "test.db" + db.init_db(path) + with db.get_db(path) as conn: + db.insert_run( + conn, + { + "run_id": "test-run-no-profile", + "pipeline_version": "v0.0.1", + }, + ) + conn2 = sqlite3.connect(str(path)) + try: + row = conn2.execute("SELECT profile FROM pipeline_runs WHERE run_id='test-run-no-profile'").fetchone() + finally: + conn2.close() + assert row is not None + assert row[0] == "" diff --git a/generative/tests/test_per_agent_tracking.py b/generative/tests/test_per_agent_tracking.py index 9ed0db0..4b79b06 100644 --- a/generative/tests/test_per_agent_tracking.py +++ b/generative/tests/test_per_agent_tracking.py @@ -40,6 +40,22 @@ def test_trace_run_start_writes_model_config(tmp_path, monkeypatch): assert entry["run_id"] == "test-run" +def test_trace_run_start_writes_profile(tmp_path, monkeypatch): + from generative.agents.tracing import JsonlBackend, trace_run_start + import generative.agents.tracing as tracing + + backend = JsonlBackend(run_dir=tmp_path, run_id="test-run") + monkeypatch.setattr(tracing, "_backend", backend) + monkeypatch.setattr(tracing, "_RUN_ID", "test-run") + + trace_run_start({"planner": "opus"}, profile="fast") + + lines = (tmp_path / "test-run.jsonl").read_text().splitlines() + entry = json.loads(lines[0]) + assert entry["type"] == "run_start" + assert entry["profile"] == "fast" + + def test_model_config_has_required_keys(): from generative.config import MODEL_CONFIG diff --git a/shared/db_schema.py b/shared/db_schema.py index ad1927d..95c42da 100644 --- a/shared/db_schema.py +++ b/shared/db_schema.py @@ -30,7 +30,8 @@ tokens_cache_read INT DEFAULT 0, duration_s REAL DEFAULT 0, eval_version TEXT, - fully_cached INT DEFAULT 0 + fully_cached INT DEFAULT 0, + profile TEXT DEFAULT '' ); CREATE TABLE IF NOT EXISTS note_evals (