Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions generative/agents/tracing.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,14 +73,15 @@ def trace_event(agent: str, event_type: str, data: dict) -> None:
)


def trace_run_start(model_config: dict) -> None:
"""Schreibt Run-Start-Entry mit Model-Config."""
def trace_run_start(model_config: dict, profile: str = "") -> None:
"""Schreibt Run-Start-Entry mit Model-Config + aktivem Runtime-Profil (#235)."""
_backend.write(
{
"ts": time.strftime("%Y-%m-%dT%H:%M:%S"),
"type": "run_start",
"run_id": _RUN_ID,
"model_config": model_config,
"profile": profile,
}
)

Expand Down
10 changes: 7 additions & 3 deletions generative/db.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,9 @@ def init_db(path: Path = DB_PATH) -> None:
_add_column(conn, "pipeline_runs", "n_words INT DEFAULT 0")
_add_column(conn, "pipeline_runs", "model TEXT DEFAULT ''")
_add_column(conn, "pipeline_runs", "cost_usd REAL DEFAULT 0.0")
# #235: aktives Runtime-Profil (legacy/fast/balanced/quality) mitschreiben —
# bisher nur im Stdout-Log, dadurch Alt- und A/B-Läufe in der DB ununterscheidbar.
_add_column(conn, "pipeline_runs", "profile TEXT DEFAULT ''")
_add_column(conn, "note_evals", "anchor_rate REAL")
# Anker-Roh-Counts: für die gepoolte Halluzinationsrate (Σ halluziniert /
# Σ gesamt) im Dashboard — die Pipeline berechnet sie ohnehin, persistiert
Expand Down Expand Up @@ -102,7 +105,7 @@ def insert_run(conn: sqlite3.Connection, data: dict) -> None:
run_id, timestamp, pipeline_version, pdf_source, pdf_key, pdf_label,
n_generated, n_extracted, n_vault, n_inbox, n_merge, n_dropped, n_words,
model, tokens_total, tokens_input, tokens_output, tokens_cache_read,
duration_s, eval_version
duration_s, eval_version, profile

n_generated = geschriebene Notes (historische Semantik, unangetastet).
n_extracted = "nach Planner/Extractor generiert" (Funnel-Top, #197). Fehlt
Expand All @@ -115,12 +118,12 @@ def insert_run(conn: sqlite3.Connection, data: dict) -> None:
(run_id, timestamp, pipeline_version, pdf_source, pdf_key, pdf_label,
n_generated, n_extracted, n_vault, n_inbox, n_merge, n_dropped, n_words, model,
cost_usd, tokens_total, tokens_input, tokens_output, tokens_cache_read,
duration_s, eval_version, fully_cached)
duration_s, eval_version, fully_cached, profile)
VALUES
(:run_id, :timestamp, :pipeline_version, :pdf_source, :pdf_key, :pdf_label,
:n_generated, :n_extracted, :n_vault, :n_inbox, :n_merge, :n_dropped, :n_words, :model,
:cost_usd, :tokens_total, :tokens_input, :tokens_output, :tokens_cache_read,
:duration_s, :eval_version, :fully_cached)
:duration_s, :eval_version, :fully_cached, :profile)
""",
{
"run_id": data.get("run_id"),
Expand All @@ -145,6 +148,7 @@ def insert_run(conn: sqlite3.Connection, data: dict) -> None:
"duration_s": data.get("duration_s", 0.0),
"eval_version": data.get("eval_version"),
"fully_cached": 1 if (data.get("tokens_total", 0) == 0 and data.get("duration_s", 0) > 0) else 0,
"profile": data.get("profile", ""),
},
)

Expand Down
3 changes: 2 additions & 1 deletion generative/orchestrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -1917,7 +1917,7 @@ def _run_extraction_stages(
from generative.agents.base import trace_run_start as _trace_run_start
from generative.config import MODEL_CONFIG as _MODEL_CONFIG

_trace_run_start(_MODEL_CONFIG)
_trace_run_start(_MODEL_CONFIG, profile=getattr(runtime_config, "profile", ""))

# --- Schritt 1: PDF → Text + Metadata → Chunks ---
print("[1/7] PDF extrahieren und chunken…")
Expand Down Expand Up @@ -2784,6 +2784,7 @@ def main(argv: list[str] | None = None):
"tokens_output": _tok_out,
"tokens_cache_read": _tok_cache_r,
"duration_s": _wall_s,
"profile": runtime_config.profile,
},
)
except Exception as _db_err:
Expand Down
58 changes: 58 additions & 0 deletions generative/tests/test_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,61 @@ def test_insert_run_stores_cost_usd():
assert abs(row[0] - 0.1234) < 0.0001
finally:
os.unlink(path)


# --- #235: Laufzeit-Profil persistieren ------------------------------------


def test_pipeline_runs_has_profile_column(tmp_path):
from generative import db

path = tmp_path / "test.db"
db.init_db(path)
with db.get_db(path) as conn:
cols = [r[1] for r in conn.execute("PRAGMA table_info(pipeline_runs)").fetchall()]
assert "profile" in cols


def test_insert_run_stores_profile(tmp_path):
from generative import db

path = tmp_path / "test.db"
db.init_db(path)
with db.get_db(path) as conn:
db.insert_run(
conn,
{
"run_id": "test-run-profile",
"pipeline_version": "v0.0.1",
"profile": "balanced",
},
)
conn2 = sqlite3.connect(str(path))
try:
row = conn2.execute("SELECT profile FROM pipeline_runs WHERE run_id='test-run-profile'").fetchone()
finally:
conn2.close()
assert row is not None
assert row[0] == "balanced"


def test_insert_run_profile_defaults_to_empty_string(tmp_path):
from generative import db

path = tmp_path / "test.db"
db.init_db(path)
with db.get_db(path) as conn:
db.insert_run(
conn,
{
"run_id": "test-run-no-profile",
"pipeline_version": "v0.0.1",
},
)
conn2 = sqlite3.connect(str(path))
try:
row = conn2.execute("SELECT profile FROM pipeline_runs WHERE run_id='test-run-no-profile'").fetchone()
finally:
conn2.close()
assert row is not None
assert row[0] == ""
16 changes: 16 additions & 0 deletions generative/tests/test_per_agent_tracking.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,22 @@ def test_trace_run_start_writes_model_config(tmp_path, monkeypatch):
assert entry["run_id"] == "test-run"


def test_trace_run_start_writes_profile(tmp_path, monkeypatch):
from generative.agents.tracing import JsonlBackend, trace_run_start
import generative.agents.tracing as tracing

backend = JsonlBackend(run_dir=tmp_path, run_id="test-run")
monkeypatch.setattr(tracing, "_backend", backend)
monkeypatch.setattr(tracing, "_RUN_ID", "test-run")

trace_run_start({"planner": "opus"}, profile="fast")

lines = (tmp_path / "test-run.jsonl").read_text().splitlines()
entry = json.loads(lines[0])
assert entry["type"] == "run_start"
assert entry["profile"] == "fast"


def test_model_config_has_required_keys():
from generative.config import MODEL_CONFIG

Expand Down
3 changes: 2 additions & 1 deletion shared/db_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@
tokens_cache_read INT DEFAULT 0,
duration_s REAL DEFAULT 0,
eval_version TEXT,
fully_cached INT DEFAULT 0
fully_cached INT DEFAULT 0,
profile TEXT DEFAULT ''
);

CREATE TABLE IF NOT EXISTS note_evals (
Expand Down
Loading