From 0fa6b3df42937a172a8b6336aa54e0be5fa4e8bd Mon Sep 17 00:00:00 2001 From: Kcodess2807 Date: Tue, 28 Jul 2026 17:10:16 +0530 Subject: [PATCH 1/2] Engine: typed timestamped edges, structured GitHub graph, recency decay Co-Authored-By: Claude Fable 5 --- backend/tests/test_github_graph.py | 131 +++++++++++++++++ backend/tests/test_recency.py | 73 ++++++++++ backend/tests/test_router_scoring.py | 153 ++++++++++++++++++++ backend/tracerag/config.py | 43 +++++- backend/tracerag/curation.py | 4 +- backend/tracerag/db.py | 50 +++++-- backend/tracerag/github_graph.py | 206 +++++++++++++++++++++++++++ backend/tracerag/memory.py | 79 ++++++++++ backend/tracerag/recency.py | 44 ++++++ backend/tracerag/router.py | 49 +++++-- 10 files changed, 811 insertions(+), 21 deletions(-) create mode 100644 backend/tests/test_github_graph.py create mode 100644 backend/tests/test_recency.py create mode 100644 backend/tests/test_router_scoring.py create mode 100644 backend/tracerag/github_graph.py create mode 100644 backend/tracerag/memory.py create mode 100644 backend/tracerag/recency.py diff --git a/backend/tests/test_github_graph.py b/backend/tests/test_github_graph.py new file mode 100644 index 0000000..7429abe --- /dev/null +++ b/backend/tests/test_github_graph.py @@ -0,0 +1,131 @@ +"""GitHub payloads -> typed, timestamped edges. No DB, no embedder.""" +import pytest + +from tracerag import config +from tracerag.github_graph import GitHubGraphBuilder, parse_ts + + +class FakeDB: + """Records what the builder would write.""" + + def __init__(self): + self.nodes = {} + self.edges = [] + + def upsert_node(self, node_id, label, node_type, embedding, ts=0): + prev = self.nodes.get(node_id) + # mirror the real upsert: ts only moves forward + if prev and prev["ts"] > ts: + ts = prev["ts"] + self.nodes[node_id] = {"label": label, "type": node_type, "ts": ts} + + def add_relationship(self, a, b, confidence=None, relation=config.RELATION_CO_OCCURS, ts=0): + self.edges.append({"from": a, "to": b, "relation": relation, "ts": ts}) + + def rel(self, relation): + return [e for e in self.edges if e["relation"] == relation] + + +@pytest.fixture +def build(): + db = FakeDB() + return db, GitHubGraphBuilder(db, lambda text: [0.0] * config.EMBED_DIM, "acme/platform") + + +PR = { + "number": 4977, + "title": "chore: bump stripe-sdk 11.2 -> 12.0", + "merged_at": "2026-07-24T09:12:31Z", + "created_at": "2026-07-23T10:00:00Z", + "user": {"login": "lena"}, + "requested_reviewers": [{"login": "marco"}], + "body": "Routine upgrade. Fixes #2291 and closes #2280.", + "files": [{"filename": "services/payment/auth.py"}], +} + + +def test_parse_ts(): + assert parse_ts("2026-07-24T09:12:31Z") > 0 + assert parse_ts(None) == 0 + assert parse_ts("not a date") == 0 + + +def test_pull_request_emits_every_structural_relation(build): + db, b = build + from tracerag.github_graph import GraphStats + b.add_pull_request(PR, GraphStats()) + + authored = db.rel(config.RELATION_AUTHORED) + assert len(authored) == 1 + assert authored[0]["from"] == "person:lena" + assert authored[0]["to"] == "pr:acme/platform#4977" + + assert len(db.rel(config.RELATION_REVIEWED)) == 1 + # both "Fixes #2291" and "closes #2280" become causal edges + assert {e["to"] for e in db.rel(config.RELATION_RESOLVES)} == { + "issue:acme/platform#2291", "issue:acme/platform#2280" + } + touches = {e["to"] for e in db.rel(config.RELATION_TOUCHES)} + assert "file:acme/platform:services/payment/auth.py" in touches + + +def test_merge_time_lands_on_the_pr_node(build): + db, b = build + from tracerag.github_graph import GraphStats + b.add_pull_request(PR, GraphStats()) + node = db.nodes["pr:acme/platform#4977"] + assert node["ts"] == parse_ts("2026-07-24T09:12:31Z") # merged_at, not created_at + assert node["type"] == "PR" + + +def test_no_co_occurrence_edges_from_structured_input(build): + db, b = build + from tracerag.github_graph import GraphStats + b.add_pull_request(PR, GraphStats()) + assert db.rel(config.RELATION_CO_OCCURS) == [] + + +def test_issues_endpoint_skips_pull_requests(build): + db, b = build + from tracerag.github_graph import GraphStats + stats = GraphStats() + b.add_issue({"number": 1, "pull_request": {}, "created_at": "2026-07-01T00:00:00Z"}, stats) + assert db.edges == [] + b.add_issue({"number": 2291, "title": "sev1", "created_at": "2026-07-23T00:00:00Z", + "user": {"login": "ana"}}, stats) + assert len(db.rel(config.RELATION_REPORTED)) == 1 + + +def test_commit_author_and_closes(build): + db, b = build + from tracerag.github_graph import GraphStats + b.add_commit({ + "sha": "7a2b4159fbc", + "commit": {"message": "fix auth flow\n\nFixes #2291", + "author": {"date": "2026-07-24T08:00:00Z", "name": "Lena K."}}, + "author": {"login": "lena"}, + }, GraphStats()) + assert db.rel(config.RELATION_AUTHORED)[0]["from"] == "person:lena" + assert db.rel(config.RELATION_RESOLVES)[0]["to"] == "issue:acme/platform#2291" + assert db.nodes["commit:acme/platform:7a2b415"]["type"] == "Commit" + + +def test_build_counts_relations(build): + db, b = build + stats = b.build(pulls=[PR], issues=[], commits=[]) + assert stats.edges == len(db.edges) + assert stats.by_relation[config.RELATION_AUTHORED] == 1 + assert stats.nodes > 0 + + +def test_structural_edges_outweigh_proximity(): + """The whole point of typing edges: a real relation must beat a guess.""" + authored = config.RELATION_WEIGHTS[config.RELATION_AUTHORED] + co = config.RELATION_WEIGHTS[config.RELATION_CO_OCCURS] + assert authored > co + # even a 2-hop structural path beats a 1-hop proximity hop + assert authored * config.RELATION_WEIGHTS[config.RELATION_RESOLVES] > co + + +if __name__ == "__main__": + pytest.main([__file__, "-q"]) diff --git a/backend/tests/test_recency.py b/backend/tests/test_recency.py new file mode 100644 index 0000000..1f9b9f3 --- /dev/null +++ b/backend/tests/test_recency.py @@ -0,0 +1,73 @@ +"""Recency decay: age -> score multiplier.""" +import time + +import pytest + +from tracerag import config +from tracerag.recency import age_days, decay_factor, half_life_for + +DAY = 86400 + + +def test_no_timestamp_is_not_penalised(): + assert decay_factor(0) == 1.0 + assert decay_factor(None) == 1.0 + assert age_days(0) is None + + +def test_fresh_beats_stale(): + now = time.time() + fresh = decay_factor(int(now - 1 * DAY), "PR", now=now) + stale = decay_factor(int(now - 240 * DAY), "PR", now=now) + assert fresh > stale + assert fresh > 0.95 + + +def test_half_life_halves_the_score(): + now = time.time() + hl = half_life_for("PR") + factor = decay_factor(int(now - hl * DAY), "PR", now=now) + assert factor == pytest.approx(0.5, abs=0.01) + + +def test_type_specific_half_lives(): + now = time.time() + age = int(now - 30 * DAY) + # an incident goes stale much faster than a service definition + assert decay_factor(age, "Ticket", now=now) < decay_factor(age, "Service", now=now) + + +def test_floor_keeps_ancient_items_reachable(): + now = time.time() + ancient = decay_factor(int(now - 5000 * DAY), "Ticket", now=now) + assert ancient == pytest.approx(config.RECENCY_FLOOR) + assert ancient > 0 + + +def test_future_timestamps_do_not_boost(): + now = time.time() + assert decay_factor(int(now + 10 * DAY), "PR", now=now) == pytest.approx(1.0) + + +def test_disabled_is_a_no_op(monkeypatch): + monkeypatch.setattr(config, "RECENCY_ENABLED", False) + now = time.time() + assert decay_factor(int(now - 900 * DAY), "Ticket", now=now) == 1.0 + + +def test_the_failure_demo_scenario(): + """The story the product is built on: a stale PR that reads like the query + must lose to a fresh one once recency is applied.""" + now = time.time() + stale_relevance, fresh_relevance = 0.91, 0.34 # raw similarity + stale = stale_relevance * decay_factor(int(now - 240 * DAY), "PR", now=now) + fresh = fresh_relevance * decay_factor(int(now - 1 * DAY), "PR", now=now) + assert stale < 0.5 # 8-month-old PR is heavily discounted + assert fresh > 0.33 # yesterday's change keeps essentially all its score + # recency alone doesn't flip this one — the graph path is what closes the gap, + # but the ranking distance shrinks from 2.7x to under 1.5x + assert (stale / fresh) < 1.5 + + +if __name__ == "__main__": + pytest.main([__file__, "-q"]) diff --git a/backend/tests/test_router_scoring.py b/backend/tests/test_router_scoring.py new file mode 100644 index 0000000..2dc34b8 --- /dev/null +++ b/backend/tests/test_router_scoring.py @@ -0,0 +1,153 @@ +"""Router scoring: relation-weighted traversal + recency decay. + +Uses a fake store and a stub embedder so the ranking logic is tested on its +own — no DB driver, no model download. +""" +import time + +import pytest + +from tracerag import config +from tracerag.router import TraceRouter + +DAY = 86400 +NOW = time.time() + + +class FakeDB: + """Minimal store: vector hits, typed neighbours, no documents.""" + + def __init__(self, hits, neighbours): + self._hits = hits + self._neighbours = neighbours + + def vector_search(self, embedding, k=10): + return self._hits[:k] + + def expand_frontier(self, node_ids, k, max_degree): + return {n: self._neighbours.get(n, [])[:k] for n in node_ids + if n in self._neighbours} + + def find_nodes_by_label(self, label): + return [] + + def documents_for_entities(self, ids): + return {} + + +def make_router(hits, neighbours): + r = TraceRouter(FakeDB(hits, neighbours)) + r._encode = lambda text: [0.0] * config.EMBED_DIM # skip the real embedder + return r + + +def hit(node_id, sim, ntype="PR", ts=0, label=None): + return {"id": node_id, "label": label or node_id, "type": ntype, + "similarity": sim, "ts": ts} + + +def nbr(node_id, relation, ntype="PR", ts=0): + return {"id": node_id, "label": node_id, "type": ntype, "ts": ts, + "relation": relation, + "confidence": config.RELATION_WEIGHTS[relation]} + + +def test_structural_hop_outranks_co_occurrence(): + """A node reached through AUTHORED must beat one reached by proximity.""" + router = make_router( + hits=[hit("seed", 0.9)], + neighbours={"seed": [nbr("real", config.RELATION_AUTHORED), + nbr("noise", config.RELATION_CO_OCCURS)]}, + ) + res = router.route("who owns this?", top_k=10) + scores = {r.id: r.score_graph for r in res.results} + assert scores["real"] > scores["noise"] + # and the gap is the relation weight ratio, not an accident + assert scores["real"] / scores["noise"] == pytest.approx( + config.RELATION_WEIGHTS[config.RELATION_AUTHORED] + / config.RELATION_WEIGHTS[config.RELATION_CO_OCCURS], rel=1e-3 + ) + + +def test_two_hop_structural_beats_one_hop_proximity(): + router = make_router( + hits=[hit("seed", 0.9)], + neighbours={ + "seed": [nbr("pr", config.RELATION_AUTHORED), + nbr("noise", config.RELATION_CO_OCCURS)], + "pr": [nbr("issue", config.RELATION_RESOLVES, ntype="Ticket")], + }, + ) + res = router.route("which issue did that fix?", top_k=10) + scores = {r.id: r.score_graph for r in res.results} + assert scores["issue"] > scores["noise"] + + +def test_graph_scores_are_no_longer_all_one(): + """Regression: with untyped 1.0 edges every reachable node tied at 1.0.""" + router = make_router( + hits=[hit("seed", 0.9)], + neighbours={ + "seed": [nbr("a", config.RELATION_AUTHORED), + nbr("b", config.RELATION_TOUCHES), + nbr("c", config.RELATION_CO_OCCURS)], + }, + ) + res = router.route("who owns this?", top_k=10) + graph_only = {r.score_graph for r in res.results if r.id != "seed"} + assert len(graph_only) > 1, "traversal produced no ranking gradient" + + +def test_recency_demotes_a_stale_but_similar_hit(): + """The failure-demo scenario, end to end through the router.""" + stale = hit("pr_stale", 0.91, ts=int(NOW - 240 * DAY)) + fresh = hit("pr_fresh", 0.60, ts=int(NOW - 1 * DAY)) + router = make_router(hits=[stale, fresh], neighbours={}) + res = router.route("explain what broke", top_k=10) + ranked = [r.id for r in res.results] + assert ranked[0] == "pr_fresh", f"stale hit still won: {ranked}" + + +def test_undated_nodes_are_not_penalised(): + router = make_router( + hits=[hit("dated", 0.8, ts=int(NOW - 300 * DAY)), hit("undated", 0.8, ts=0)], + neighbours={}, + ) + res = router.route("explain the architecture", top_k=10) + by_id = {r.id: r for r in res.results} + assert by_id["undated"].recency == 1.0 + assert by_id["undated"].age_days is None + assert by_id["dated"].recency < 1.0 + assert by_id["undated"].score_total > by_id["dated"].score_total + + +def test_trace_log_exposes_relations_and_recency(): + """The viewer can only show why something ranked if the trace says so.""" + router = make_router( + hits=[hit("seed", 0.9, ts=int(NOW - 10 * DAY))], + neighbours={"seed": [nbr("pr", config.RELATION_RESOLVES)]}, + ) + res = router.route("who fixed it?", top_k=10) + + hops = res.trace_log["execution_path"]["graph_hops"] + assert hops and hops[0]["relation"] == config.RELATION_RESOLVES + + recency = res.trace_log["recency"] + assert recency["enabled"] is True + applied = {a["id"]: a for a in recency["applied"]} + assert applied["seed"]["age_days"] == pytest.approx(10, abs=0.5) + assert 0 < applied["seed"]["factor"] < 1 + + +def test_recency_can_be_disabled(monkeypatch): + monkeypatch.setattr(config, "RECENCY_ENABLED", False) + router = make_router( + hits=[hit("stale", 0.91, ts=int(NOW - 400 * DAY)), hit("fresh", 0.60, ts=int(NOW))], + neighbours={}, + ) + res = router.route("explain what broke", top_k=10) + assert res.results[0].id == "stale" # pure similarity ordering restored + + +if __name__ == "__main__": + pytest.main([__file__, "-q"]) diff --git a/backend/tracerag/config.py b/backend/tracerag/config.py index c6d7bcf..f9ba7c1 100644 --- a/backend/tracerag/config.py +++ b/backend/tracerag/config.py @@ -199,8 +199,49 @@ # ladybugdb schema (kùzu-compatible cypher ddl) NODE_TABLE = "Entity" -REL_TABLE = "RELATES_TO" # entity -> entity, same-window co-occurrence +REL_TABLE = "RELATES_TO" # entity -> entity; carries relation + ts DOC_TABLE = "Document" # source doc node, no embedding MENTIONS_TABLE = "MENTIONS" # document -> entity VECTOR_INDEX = "idx_entity_embedding" + +# Relation vocabulary. Structured sources (the GitHub API) emit these directly; +# CO_OCCURS is the weak fallback for text we can only infer from proximity. +RELATION_AUTHORED = "AUTHORED" +RELATION_REVIEWED = "REVIEWED" +RELATION_RESOLVES = "RESOLVES" +RELATION_TOUCHES = "TOUCHES" +RELATION_PART_OF = "PART_OF" +RELATION_REPORTED = "REPORTED" +RELATION_CO_OCCURS = "CO_OCCURS" + +# How much a hop through each relation is worth. Traversal multiplies these, so +# a 2-hop path through two strong edges still beats one weak hop. +RELATION_WEIGHTS = { + RELATION_AUTHORED: 0.95, + RELATION_RESOLVES: 0.92, + RELATION_REVIEWED: 0.85, + RELATION_TOUCHES: 0.80, + RELATION_PART_OF: 0.80, + RELATION_REPORTED: 0.75, + RELATION_CO_OCCURS: 0.35, # proximity is a hint, not a fact +} +DEFAULT_RELATION_WEIGHT = float(os.getenv("TRACERAG_DEFAULT_REL_WEIGHT", "0.5")) + +# Recency: score *= 0.5 ** (age_days / half_life) — at half_life days an item +# keeps half its score. Incidents go stale in weeks, service definitions in a +# year, so the half-life is per entity type. +RECENCY_ENABLED = os.getenv("TRACERAG_RECENCY", "1") not in ("0", "false", "False") +RECENCY_HALF_LIFE_DAYS = { + "Ticket": 21.0, + "Issue": 21.0, + "PR": 60.0, + "Commit": 45.0, + "Person": 180.0, + "File": 120.0, + "Service": 365.0, + "Repo": 365.0, +} +DEFAULT_HALF_LIFE_DAYS = float(os.getenv("TRACERAG_HALF_LIFE_DAYS", "90")) +# Never decay an item to nothing — old but on-topic still beats unrelated. +RECENCY_FLOOR = float(os.getenv("TRACERAG_RECENCY_FLOOR", "0.35")) VECTOR_METRIC = "cosine" diff --git a/backend/tracerag/curation.py b/backend/tracerag/curation.py index 456e920..c20f43d 100644 --- a/backend/tracerag/curation.py +++ b/backend/tracerag/curation.py @@ -213,5 +213,7 @@ def _build_chunks_and_edges( if a == b or (a, b) in seen_pairs: continue seen_pairs.add((a, b)) - self.db.add_relationship(a, b) + # proximity only — tagged CO_OCCURS so traversal weights it far + # below a structural edge, and upgrades it if one arrives later + self.db.add_relationship(a, b, relation=config.RELATION_CO_OCCURS) stats.relates_edges += 1 diff --git a/backend/tracerag/db.py b/backend/tracerag/db.py index 472374b..f24091c 100644 --- a/backend/tracerag/db.py +++ b/backend/tracerag/db.py @@ -109,9 +109,10 @@ def _fetch( return self._records(conn.execute(query, params or {})) def init_schema(self) -> None: + # ts = unix seconds of the underlying event (merge/creation), 0 when unknown self.execute( f"CREATE NODE TABLE IF NOT EXISTS {config.NODE_TABLE} (" - f"id STRING PRIMARY KEY, label STRING, type STRING, " + f"id STRING PRIMARY KEY, label STRING, type STRING, ts INT64, " f"embedding FLOAT[{config.EMBED_DIM}]);" ) self.execute( @@ -120,7 +121,8 @@ def init_schema(self) -> None: ) self.execute( f"CREATE REL TABLE IF NOT EXISTS {config.REL_TABLE} (" - f"FROM {config.NODE_TABLE} TO {config.NODE_TABLE}, confidence DOUBLE);" + f"FROM {config.NODE_TABLE} TO {config.NODE_TABLE}, " + f"confidence DOUBLE, relation STRING, ts INT64);" ) self.execute( f"CREATE REL TABLE IF NOT EXISTS {config.MENTIONS_TABLE} (" @@ -152,22 +154,46 @@ def upsert_node( label: str, node_type: str, embedding: Sequence[float], + ts: int = 0, ) -> None: - # no ON MATCH SET: never clobber a canonical node with a noisier surface form + # no ON MATCH SET on label/type: never clobber a canonical node with a + # noisier surface form. ts does move forward — recency is the newest fact. self._check_dim(embedding) self.execute( f"MERGE (e:{config.NODE_TABLE} {{id: $id}}) " - f"ON CREATE SET e.label = $label, e.type = $type, e.embedding = $embedding", - {"id": node_id, "label": label, "type": node_type, + f"ON CREATE SET e.label = $label, e.type = $type, e.ts = $ts, " + f"e.embedding = $embedding " + f"ON MATCH SET e.ts = CASE WHEN $ts > e.ts THEN $ts ELSE e.ts END", + {"id": node_id, "label": label, "type": node_type, "ts": int(ts), "embedding": list(embedding)}, ) - def add_relationship(self, from_id: str, to_id: str, confidence: float = 1.0) -> None: + def add_relationship( + self, + from_id: str, + to_id: str, + confidence: float | None = None, + relation: str = config.RELATION_CO_OCCURS, + ts: int = 0, + ) -> None: + """Typed edge. Confidence defaults to the relation's configured weight, + so a structural AUTHORED hop outranks a proximity guess.""" + if confidence is None: + confidence = config.RELATION_WEIGHTS.get( + relation, config.DEFAULT_RELATION_WEIGHT + ) self.execute( f"MATCH (a:{config.NODE_TABLE} {{id: $from_id}}), " f"(b:{config.NODE_TABLE} {{id: $to_id}}) " - f"MERGE (a)-[r:{config.REL_TABLE}]->(b) ON CREATE SET r.confidence = $confidence", - {"from_id": from_id, "to_id": to_id, "confidence": confidence}, + f"MERGE (a)-[r:{config.REL_TABLE}]->(b) " + f"ON CREATE SET r.confidence = $confidence, r.relation = $relation, r.ts = $ts " + # a stronger relation later (CO_OCCURS then AUTHORED) upgrades the edge + f"ON MATCH SET r.confidence = CASE WHEN $confidence > r.confidence " + f"THEN $confidence ELSE r.confidence END, " + f"r.relation = CASE WHEN $confidence > r.confidence THEN $relation " + f"ELSE r.relation END", + {"from_id": from_id, "to_id": to_id, "confidence": float(confidence), + "relation": relation, "ts": int(ts)}, ) def upsert_document( @@ -215,7 +241,7 @@ def vector_search( f"CALL QUERY_VECTOR_INDEX('{config.NODE_TABLE}', " f"'{config.VECTOR_INDEX}', $q, {int(k)}) " f"RETURN node.id AS id, node.label AS label, node.type AS type, " - f"distance ORDER BY distance;", + f"node.ts AS ts, distance ORDER BY distance;", {"q": list(query_embedding)}, ) rows = [] @@ -223,6 +249,7 @@ def vector_search( distance = float(r["distance"]) rows.append({ "id": r["id"], "label": r["label"], "type": r["type"], + "ts": int(r["ts"]) if r.get("ts") is not None else 0, "distance": distance, "similarity": 1.0 - distance, }) return rows @@ -259,7 +286,8 @@ def expand_frontier( f"MATCH (a:{config.NODE_TABLE}) WHERE a.id IN $ids " f"OPTIONAL MATCH (a)-[r:{config.REL_TABLE}]-(b:{config.NODE_TABLE}) " f"RETURN a.id AS from_id, b.id AS to_id, b.label AS label, " - f"b.type AS type, r.confidence AS confidence;", + f"b.type AS type, b.ts AS ts, r.confidence AS confidence, " + f"r.relation AS relation;", {"ids": list(node_ids)}, ) grouped: dict[str, list[dict[str, Any]]] = {} @@ -268,6 +296,8 @@ def expand_frontier( continue grouped.setdefault(r["from_id"], []).append({ "id": r["to_id"], "label": r["label"], "type": r["type"], + "ts": int(r["ts"]) if r.get("ts") is not None else 0, + "relation": r.get("relation") or config.RELATION_CO_OCCURS, "confidence": float(r["confidence"]) if r["confidence"] is not None else 1.0, }) diff --git a/backend/tracerag/github_graph.py b/backend/tracerag/github_graph.py new file mode 100644 index 0000000..39e794c --- /dev/null +++ b/backend/tracerag/github_graph.py @@ -0,0 +1,206 @@ +"""Build the memory graph straight from GitHub's structured payloads. + +The API already knows who authored a PR, which issue it closes, and which +files it touched — so we read those relations instead of guessing them from +word proximity. Every edge here is a fact from the payload, never an +inference, and every node carries the event's timestamp for recency scoring. + +Text extraction (extract.py + curation.py) still runs for prose; this module +covers the part of the graph that structure can answer exactly. +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass, field +from datetime import datetime +from typing import TYPE_CHECKING, Any, Iterable + +from . import config + +if TYPE_CHECKING: # the builder only needs upsert_node/add_relationship + from .db import TraceDB + +logger = logging.getLogger(__name__) + +# "Fixes #982", "closes JIRA-77" in a PR body — GitHub only auto-links some of these +_CLOSES_RE = re.compile( + r"(?:fix(?:e[sd])?|close[sd]?|resolve[sd]?)\s+#(\d+)", re.IGNORECASE +) + + +@dataclass +class GraphStats: + nodes: int = 0 + edges: int = 0 + by_relation: dict[str, int] = field(default_factory=dict) + + def count(self, relation: str) -> None: + self.edges += 1 + self.by_relation[relation] = self.by_relation.get(relation, 0) + 1 + + +def parse_ts(value: str | None) -> int: + """GitHub ISO-8601 ('2026-07-24T09:12:31Z') -> unix seconds; 0 when absent.""" + if not value: + return 0 + try: + return int(datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()) + except (ValueError, AttributeError): + return 0 + + +def _login(actor: dict[str, Any] | None) -> str | None: + if not actor: + return None + return actor.get("login") or actor.get("name") + + +class GitHubGraphBuilder: + """Writes typed, timestamped edges from GitHub payloads into the graph. + + Nodes need an embedding, so the caller supplies an embed function — the + same one curation uses, keeping one vector space across both paths. + """ + + def __init__(self, db: "TraceDB", embed_fn, repo: str) -> None: + self.db = db + self.embed = embed_fn + self.repo = repo + self.repo_id = f"repo:{repo}" + self._seen: set[str] = set() + + # nodes ------------------------------------------------------------- + def _node(self, node_id: str, label: str, node_type: str, ts: int = 0) -> str: + """Upsert once per run; embeddings are the expensive part.""" + if node_id not in self._seen: + self.db.upsert_node(node_id, label, node_type, self.embed(label), ts=ts) + self._seen.add(node_id) + elif ts: + # already embedded this run — still let a newer event move the clock + self.db.upsert_node(node_id, label, node_type, self.embed(label), ts=ts) + return node_id + + def person(self, login: str) -> str: + return self._node(f"person:{login}", login, "Person") + + def pull_request(self, number: int, title: str, ts: int) -> str: + return self._node(f"pr:{self.repo}#{number}", f"PR #{number}", "PR", ts) + + def issue(self, number: int, title: str = "", ts: int = 0) -> str: + return self._node(f"issue:{self.repo}#{number}", f"Issue #{number}", "Ticket", ts) + + def file(self, path: str) -> str: + return self._node(f"file:{self.repo}:{path}", path, "File") + + def commit(self, sha: str, ts: int) -> str: + return self._node(f"commit:{self.repo}:{sha[:7]}", sha[:7], "Commit", ts) + + def repository(self) -> str: + return self._node(self.repo_id, self.repo.split("/")[-1], "Repo") + + # ingestion --------------------------------------------------------- + def add_pull_request(self, pr: dict[str, Any], stats: GraphStats) -> None: + number = pr.get("number") + if number is None: + return + ts = parse_ts(pr.get("merged_at") or pr.get("closed_at") or pr.get("created_at")) + pr_id = self.pull_request(number, pr.get("title") or "", ts) + repo_id = self.repository() + self.db.add_relationship(pr_id, repo_id, relation=config.RELATION_TOUCHES, ts=ts) + stats.count(config.RELATION_TOUCHES) + + author = _login(pr.get("user")) + if author: + self.db.add_relationship( + self.person(author), pr_id, relation=config.RELATION_AUTHORED, ts=ts + ) + stats.count(config.RELATION_AUTHORED) + + for reviewer in pr.get("requested_reviewers") or []: + login = _login(reviewer) + if login: + self.db.add_relationship( + self.person(login), pr_id, relation=config.RELATION_REVIEWED, ts=ts + ) + stats.count(config.RELATION_REVIEWED) + + # closes/fixes references — the causal link that makes tracing possible + for ref in set(_CLOSES_RE.findall(pr.get("body") or "")): + self.db.add_relationship( + pr_id, self.issue(int(ref)), relation=config.RELATION_RESOLVES, ts=ts + ) + stats.count(config.RELATION_RESOLVES) + + for f in pr.get("files") or []: + path = f.get("filename") if isinstance(f, dict) else f + if path: + self.db.add_relationship( + pr_id, self.file(path), relation=config.RELATION_TOUCHES, ts=ts + ) + stats.count(config.RELATION_TOUCHES) + + def add_issue(self, issue: dict[str, Any], stats: GraphStats) -> None: + if issue.get("number") is None or "pull_request" in issue: + return # the issues endpoint also returns PRs + ts = parse_ts(issue.get("created_at")) + issue_id = self.issue(issue["number"], issue.get("title") or "", ts) + self.db.add_relationship( + issue_id, self.repository(), relation=config.RELATION_TOUCHES, ts=ts + ) + stats.count(config.RELATION_TOUCHES) + reporter = _login(issue.get("user")) + if reporter: + self.db.add_relationship( + self.person(reporter), issue_id, + relation=config.RELATION_REPORTED, ts=ts, + ) + stats.count(config.RELATION_REPORTED) + + def add_commit(self, commit: dict[str, Any], stats: GraphStats) -> None: + sha = commit.get("sha") + if not sha: + return + inner = commit.get("commit") or {} + ts = parse_ts((inner.get("author") or {}).get("date")) + commit_id = self.commit(sha, ts) + self.db.add_relationship( + commit_id, self.repository(), relation=config.RELATION_PART_OF, ts=ts + ) + stats.count(config.RELATION_PART_OF) + + author = _login(commit.get("author")) or (inner.get("author") or {}).get("name") + if author: + self.db.add_relationship( + self.person(author), commit_id, + relation=config.RELATION_AUTHORED, ts=ts, + ) + stats.count(config.RELATION_AUTHORED) + + for ref in set(_CLOSES_RE.findall(inner.get("message") or "")): + self.db.add_relationship( + commit_id, self.issue(int(ref)), + relation=config.RELATION_RESOLVES, ts=ts, + ) + stats.count(config.RELATION_RESOLVES) + + def build( + self, + pulls: Iterable[dict[str, Any]] = (), + issues: Iterable[dict[str, Any]] = (), + commits: Iterable[dict[str, Any]] = (), + ) -> GraphStats: + stats = GraphStats() + for pr in pulls: + self.add_pull_request(pr, stats) + for issue in issues: + self.add_issue(issue, stats) + for commit in commits: + self.add_commit(commit, stats) + stats.nodes = len(self._seen) + logger.info( + "github graph: %d nodes, %d edges %s", + stats.nodes, stats.edges, stats.by_relation, + ) + return stats diff --git a/backend/tracerag/memory.py b/backend/tracerag/memory.py new file mode 100644 index 0000000..819bf67 --- /dev/null +++ b/backend/tracerag/memory.py @@ -0,0 +1,79 @@ +"""GraphMemory — the embedded entry point. + + mem = GraphMemory("memory.lbug") + mem.ingest_github("acme/platform", pulls=prs, issues=issues, commits=commits) + hits = mem.query("who broke checkout yesterday?") + +One object owns the store, the embedder, and the router. Everything runs in +this process against a single file; nothing is sent anywhere. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any, Iterable + +from . import config +from .db import TraceDB +from .github_graph import GitHubGraphBuilder, GraphStats +from .router import RouterResponse, TraceRouter + +logger = logging.getLogger(__name__) + + +class GraphMemory: + def __init__(self, path: str | Path = config.DB_PATH, embed_model: str | None = None): + self.path = str(path) + self.db = TraceDB(self.path) + self.db.init_schema() + self.router = TraceRouter(self.db, embed_model or config.EMBED_MODEL) + self._indexed = False + + # ingest ------------------------------------------------------------ + def ingest_github( + self, + repo: str, + pulls: Iterable[dict[str, Any]] = (), + issues: Iterable[dict[str, Any]] = (), + commits: Iterable[dict[str, Any]] = (), + ) -> GraphStats: + """Typed edges from GitHub payloads — authorship, closes, touches.""" + builder = GitHubGraphBuilder(self.db, self.router._encode, repo) + stats = builder.build(pulls=pulls, issues=issues, commits=commits) + self._indexed = False + return stats + + def ingest_text(self, doc_id: str, text: str, source: str | None = None): + """Prose path: extract entities, resolve them, store co-occurrence edges.""" + from .curation import CurationEngine + from .extract import EntityExtractor + + entities = EntityExtractor().extract(text) + stats = CurationEngine(self.db).ingest(doc_id, text, entities, source=source) + self._indexed = False + return stats + + def build_index(self) -> None: + """(Re)build the HNSW index. Required before the first query after ingest.""" + self.db.build_vector_index() + self._indexed = True + + # query ------------------------------------------------------------- + def query(self, question: str, top_k: int | None = None) -> RouterResponse: + if not self._indexed: + self.build_index() + return self.router.route(question, top_k) + + def context(self, question: str, top_k: int | None = None) -> str: + """Retrieved context as a single grounded string, ready for a prompt.""" + return self.router.build_context(self.query(question, top_k).results) + + def close(self) -> None: + self.db.close() + + def __enter__(self) -> "GraphMemory": + return self + + def __exit__(self, *exc: object) -> None: + self.close() diff --git a/backend/tracerag/recency.py b/backend/tracerag/recency.py new file mode 100644 index 0000000..92490b4 --- /dev/null +++ b/backend/tracerag/recency.py @@ -0,0 +1,44 @@ +"""Age-based score decay: score *= 0.5 ** (age_days / half_life), floored. + +A true half-life — at exactly half_life days the multiplier is 0.5 — so the +config numbers mean what they say when you tune them. + +Half-life is per entity type — an incident goes stale in weeks, a service +definition in a year. Items with no timestamp are left untouched rather than +penalised, so a graph without dates behaves exactly as it did before. +""" + +from __future__ import annotations + +import time + +from . import config + +SECONDS_PER_DAY = 86400.0 + + +def age_days(ts: int | None, now: float | None = None) -> float | None: + """Age in days, or None when the item carries no usable timestamp.""" + if not ts: + return None + now = time.time() if now is None else now + return max(0.0, (now - float(ts)) / SECONDS_PER_DAY) + + +def half_life_for(node_type: str | None) -> float: + return config.RECENCY_HALF_LIFE_DAYS.get( + node_type or "", config.DEFAULT_HALF_LIFE_DAYS + ) + + +def decay_factor( + ts: int | None, node_type: str | None = None, now: float | None = None +) -> float: + """Multiplier in [RECENCY_FLOOR, 1.0]. 1.0 when recency is off or ts is unknown.""" + if not config.RECENCY_ENABLED: + return 1.0 + age = age_days(ts, now) + if age is None: + return 1.0 + factor = 0.5 ** (age / half_life_for(node_type)) + return max(config.RECENCY_FLOOR, factor) diff --git a/backend/tracerag/router.py b/backend/tracerag/router.py index 286dd91..7f4cdc9 100644 --- a/backend/tracerag/router.py +++ b/backend/tracerag/router.py @@ -8,8 +8,13 @@ from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field +from typing import TYPE_CHECKING + from . import config -from .db import TraceDB +from .recency import age_days, decay_factor + +if TYPE_CHECKING: # keeps the scoring logic importable without the DB driver + from .db import TraceDB logger = logging.getLogger(__name__) @@ -39,6 +44,8 @@ class RoutedNode: score_total: float score_vector: float score_graph: float + recency: float = 1.0 # age multiplier applied to score_total + age_days: float | None = None # None when the node carries no timestamp documents: list[dict] = field(default_factory=list) @@ -52,7 +59,7 @@ class RouterResponse: class TraceRouter: """Hybrid retriever: vector + graph, fused by query intent.""" - def __init__(self, db: TraceDB, embed_model: str = config.EMBED_MODEL) -> None: + def __init__(self, db: "TraceDB", embed_model: str = config.EMBED_MODEL) -> None: self.db = db self.embed_model = embed_model self._embedder = None @@ -190,7 +197,8 @@ def _vector_hits( nid = h["id"] if nid is None: continue - meta.setdefault(nid, {"label": h.get("label"), "type": h.get("type")}) + meta.setdefault(nid, {"label": h.get("label"), "type": h.get("type"), + "ts": h.get("ts", 0)}) out.append((nid, float(h["similarity"]))) return out @@ -200,7 +208,9 @@ def _graph_stream( s_g: dict[str, float] = {seed: 1.0 for seed in seeds} hops: list[dict] = [] - # multiplicative BFS: path score = product of edge confidences + # multiplicative BFS: path score = product of edge weights. A hop's weight + # is the relation's own strength (AUTHORED 0.95 vs CO_OCCURS 0.35), so a + # structural two-hop path beats a proximity guess one hop away. frontier: dict[str, float] = {seed: 1.0 for seed in seeds} visited: set[str] = set(seeds) for _ in range(config.GRAPH_MAX_HOPS): @@ -213,10 +223,15 @@ def _graph_stream( for node_id, acc in frontier.items(): for nb in expanded.get(node_id, []): to_id = nb["id"] + relation = nb.get("relation") or config.RELATION_CO_OCCURS conf = nb["confidence"] path_score = acc * conf - hops.append({"from_id": node_id, "to_id": to_id, "confidence": conf}) - meta.setdefault(to_id, {"label": nb["label"], "type": nb["type"]}) + hops.append({ + "from_id": node_id, "to_id": to_id, + "confidence": conf, "relation": relation, + }) + meta.setdefault(to_id, {"label": nb["label"], "type": nb["type"], + "ts": nb.get("ts", 0)}) if path_score > s_g.get(to_id, 0.0): s_g[to_id] = path_score if to_id not in visited: @@ -302,13 +317,20 @@ def route(self, query: str, top_k: int | None = None) -> RouterResponse: sv = vector_scores.get(nid, 0.0) sg = graph_scores.get(nid, 0.0) info = meta.get(nid, {}) + ts = info.get("ts") or 0 + ntype = info.get("type") + # recency multiplies the fused score: yesterday's change outranks a + # stale one that merely reads like the query + decay = decay_factor(ts, ntype) results.append(RoutedNode( id=nid, label=info.get("label"), - type=info.get("type"), - score_total=alpha * sv + beta * sg, + type=ntype, + score_total=(alpha * sv + beta * sg) * decay, score_vector=sv, score_graph=sg, + recency=round(decay, 4), + age_days=(lambda a: round(a, 1) if a is not None else None)(age_days(ts)), )) results.sort(key=lambda r: r.score_total, reverse=True) results = results[:total_k] @@ -330,7 +352,16 @@ def route(self, query: str, top_k: int | None = None) -> RouterResponse: "execution_path": { "linked_seeds": linked_seeds, "vector_seeds": seeds, - "graph_hops": hops, + "graph_hops": hops, # each hop carries its relation + }, + "recency": { + "enabled": config.RECENCY_ENABLED, + "floor": config.RECENCY_FLOOR, + # per-node age + multiplier, so the viewer can show *why* something ranked + "applied": [ + {"id": r.id, "age_days": r.age_days, "factor": r.recency} + for r in results if r.age_days is not None + ], }, "metrics": { "graph_hits": graph_hits, From c789402073482995d81b50ce22a11b696d94b7cc Mon Sep 17 00:00:00 2001 From: Kcodess2807 Date: Tue, 28 Jul 2026 18:24:34 +0530 Subject: [PATCH 2/2] Engine: prove on a real store, migrate old ones, wire live ingest, surface in UI Running the 0.3 schema against LadybugDB for the first time found two bugs that reading could not: - edge upgrade never fired. SET items apply left to right, so r.confidence was already the new value when r.relation's guard compared against it. Relation is now decided before confidence is overwritten. - CAST($ts AS INT64). The driver infers a small $ts (0 for undated items) as INT8, so re-ingesting an undated item over a dated one overflowed. Existing stores now migrate in place instead of needing deletion: init_schema adds the missing columns with ALTER ... DEFAULT, keeping every row. It also reprices legacy edges, which the old writer stored at a degenerate 1.0 -- above every typed relation's weight, so they would have outranked real structure and could never be upgraded. ingest_github.py runs GitHubGraphBuilder over the payloads it already fetches, plus --issues / --commits / --files, so a live repo produces typed edges. The Studio renders both signals: relation labels on traced edges, an age chip and recency multiplier on nodes, and a stepper that names the relations walked and the nodes age demoted. 43 engine tests, now in CI on a torch-free dependency set. --- .github/workflows/ci.yml | 23 ++ README.md | 26 +- backend/scripts/ingest_github.py | 151 +++++++--- backend/tests/test_db_integration.py | 279 +++++++++++++++++++ backend/tests/test_ingest_github_wiring.py | 136 +++++++++ backend/tracerag/curation.py | 5 + backend/tracerag/db.py | 98 ++++++- frontend/src/components/right/EntityNode.tsx | 37 +++ frontend/src/lib/api.ts | 48 +++- frontend/src/types/trace.ts | 4 + 10 files changed, 758 insertions(+), 49 deletions(-) create mode 100644 backend/tests/test_db_integration.py create mode 100644 backend/tests/test_ingest_github_wiring.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4b7562..293ca08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,29 @@ jobs: - name: run tests run: pytest graphsight-langgraph/tests graphsight/tests -q + # The graph engine: scoring maths plus the real LadybugDB path (schema, + # migration, MERGE semantics). Deliberately avoids torch/spacy — these tests + # stub the embedder, so the whole job installs in seconds. + engine-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: install engine test deps + run: pip install pytest ladybug numpy pandas requests tqdm + # named explicitly: the rest of backend/tests needs the full API stack + - name: run engine tests + run: > + pytest -q + tests/test_recency.py + tests/test_router_scoring.py + tests/test_github_graph.py + tests/test_db_integration.py + tests/test_ingest_github_wiring.py + working-directory: backend + frontend-build: runs-on: ubuntu-latest steps: diff --git a/README.md b/README.md index 5e57e74..79c0ac4 100644 --- a/README.md +++ b/README.md @@ -285,10 +285,25 @@ hf-space/ # HuggingFace Space deployment (own git remote) ### Data model -A single generic `Entity` node table (`id, label, type, embedding`) keeps the schema dynamic +A single generic `Entity` node table (`id, label, type, ts, embedding`) keeps the schema dynamic across all entity labels. `Document` nodes store one **sliding-window chunk** each (with raw `content` for generation). Edges: `Document -[MENTIONS]-> Entity` and `Entity -[RELATES_TO]-> -Entity` (only between entities co-occurring in the *same* window — no document-wide hairball). +Entity`, carrying `confidence, relation, ts`. + +`relation` is one of `AUTHORED · RESOLVES · REVIEWED · TOUCHES · PART_OF · REPORTED · +CO_OCCURS`, each with its own weight (`config.RELATION_WEIGHTS`). Text ingestion can only +produce `CO_OCCURS` — same-window proximity, weighted lowest. Structured ingestion reads the +rest from the source API, so a two-hop `AUTHORED → RESOLVES` chain outranks a one-hop guess. +An edge **upgrades** if structure later proves a guess right; it never downgrades. + +`ts` is the underlying event time (0 when unknown), which drives recency decay: +`score *= 0.5 ** (age_days / half_life)`, floored at `RECENCY_FLOOR`. Undated entities are +left alone rather than penalised. + +**Opening an older store migrates it in place.** `init_schema()` adds the missing columns via +`ALTER TABLE ... ADD ... DEFAULT`, keeping every existing row, and reprices legacy edges — +the pre-0.3 writer stored them all at a degenerate `1.0`, which both flattened traversal and +would have outranked every typed relation. Reingest to populate real timestamps and relations. ### High-Level Design (HLD) @@ -927,6 +942,13 @@ python -m spacy download en_core_web_sm # fallback NER # 2. ingest your datasets/ (writes memory.lbug, builds the HNSW index) python scripts/ingest.py --datasets ./datasets --reset +# 2b. or ingest a GitHub repo. Two passes land in one graph: PR prose through +# extraction, and the same payloads through GitHubGraphBuilder, which reads +# authorship, reviews, "Fixes #N" and touched files as typed, dated edges. +python scripts/ingest_github.py --repo pallets/click --issues 50 --commits 50 --files +# --files costs one extra API request per PR; without it there are no +# TOUCHES edges. --no-text skips extraction and writes structure only. + # 3. evaluate (optional) python scripts/benchmark.py # -> results.csv + summary table diff --git a/backend/scripts/ingest_github.py b/backend/scripts/ingest_github.py index 17e4644..5cca3c6 100644 --- a/backend/scripts/ingest_github.py +++ b/backend/scripts/ingest_github.py @@ -2,6 +2,14 @@ python scripts/ingest_github.py --repo langchain-ai/langchain python scripts/ingest_github.py --repo owner/repo --limit 100 --reset + python scripts/ingest_github.py --repo owner/repo --issues 50 --commits 50 --files + +Two passes write into the same graph: + + * text — PR prose through extraction + curation (fuzzy CO_OCCURS edges) + * structure — the same payloads through GitHubGraphBuilder, which reads + authorship, reviews, "Fixes #N" and touched files as typed, timestamped + edges. These are facts from the API, not guesses from word proximity. Set GITHUB_TOKEN in .env to raise the rate limit from 60/hr to 5000/hr. """ @@ -26,6 +34,7 @@ from tracerag.db import TraceDB # noqa: E402 from tracerag.extract import EntityExtractor # noqa: E402 from tracerag.curation import CurationEngine, IngestStats # noqa: E402 +from tracerag.github_graph import GitHubGraphBuilder # noqa: E402 from ingest import ingest_text # noqa: E402 logger = logging.getLogger("tracerag.github") @@ -59,34 +68,71 @@ def assemble_text(pr: dict) -> tuple[str, str]: return f"pr-{number}", text -def fetch_merged_prs(repo: str, limit: int, token: str | None) -> list[dict]: - """Page through closed PRs, keeping only merged ones (merged_at != null), until limit.""" - headers = { +def _headers(token: str | None) -> dict[str, str]: + h = { "Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28", } if token: - headers["Authorization"] = f"Bearer {token}" + h["Authorization"] = f"Bearer {token}" + return h + + +def _get(url: str, token: str | None, params: dict | None = None): + resp = requests.get(url, headers=_headers(token), params=params, timeout=30) + if resp.status_code != 200: + raise RuntimeError(f"GitHub API {resp.status_code} for {url}: {resp.text[:200]}") + return resp.json() + - merged: list[dict] = [] +def _paged(url: str, limit: int, token: str | None, params: dict, + keep=lambda item: True) -> list[dict]: + """Page an endpoint until `limit` kept items, or the listing runs out.""" + out: list[dict] = [] page = 1 - while len(merged) < limit: - resp = requests.get( - f"{GITHUB_API}/repos/{repo}/pulls", - headers=headers, - params={"state": "closed", "per_page": _PER_PAGE, "page": page}, - timeout=30, - ) - if resp.status_code != 200: - raise RuntimeError( - f"GitHub API {resp.status_code} for {repo}: {resp.text[:200]}" - ) - batch = resp.json() + while len(out) < limit: + batch = _get(url, token, {**params, "per_page": _PER_PAGE, "page": page}) if not batch: break - merged.extend(pr for pr in batch if pr.get("merged_at")) + out.extend(item for item in batch if keep(item)) page += 1 - return merged[:limit] + return out[:limit] + + +def fetch_merged_prs(repo: str, limit: int, token: str | None) -> list[dict]: + """Page through closed PRs, keeping only merged ones (merged_at != null), until limit.""" + return _paged( + f"{GITHUB_API}/repos/{repo}/pulls", limit, token, + {"state": "closed"}, keep=lambda pr: bool(pr.get("merged_at")), + ) + + +def fetch_issues(repo: str, limit: int, token: str | None) -> list[dict]: + """Issues only — the endpoint also returns PRs, which the builder skips.""" + if limit <= 0: + return [] + return _paged( + f"{GITHUB_API}/repos/{repo}/issues", limit, token, + {"state": "all"}, keep=lambda i: "pull_request" not in i, + ) + + +def fetch_commits(repo: str, limit: int, token: str | None) -> list[dict]: + if limit <= 0: + return [] + return _paged(f"{GITHUB_API}/repos/{repo}/commits", limit, token, {}) + + +def attach_pr_files(repo: str, prs: list[dict], token: str | None) -> None: + """Fill each PR's `files` in place — one extra request per PR, so opt-in.""" + for pr in tqdm(prs, desc=f"{repo} files", unit="pr", leave=False): + try: + pr["files"] = _get( + f"{GITHUB_API}/repos/{repo}/pulls/{pr['number']}/files", + token, {"per_page": _PER_PAGE}, + ) + except Exception as exc: # noqa: BLE001 — a missing file list isn't fatal + logger.debug("[%s] files for #%s failed: %s", repo, pr.get("number"), exc) def repo_db_path(repo: str, graphs_dir: Path) -> Path: @@ -101,6 +147,15 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: help='One or more repos, e.g. "pallets/flask psf/requests".') p.add_argument("--limit", type=int, default=50, help="Max MERGED PRs to ingest per repo (protects rate limits).") + p.add_argument("--issues", type=int, default=0, + help="Also ingest up to N issues (REPORTED edges, RESOLVES targets).") + p.add_argument("--commits", type=int, default=0, + help="Also ingest up to N commits (AUTHORED + 'Fixes #N' edges).") + p.add_argument("--files", action="store_true", + help="Fetch each PR's changed files for TOUCHES edges. " + "Costs one extra API request per PR.") + p.add_argument("--no-text", action="store_true", + help="Skip the prose/curation pass; write structured edges only.") p.add_argument("--db", type=Path, default=None, help="Override output file (single-repo only; otherwise one " "per-repo file is created under --graphs-dir).") @@ -113,33 +168,48 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: def ingest_repo( - repo: str, db_path: Path, limit: int, reset: bool, - token: str | None, extractor: EntityExtractor, + repo: str, db_path: Path, args: argparse.Namespace, + token: str | None, extractor: EntityExtractor | None, ) -> None: - """Fetch + ingest one repo's merged PRs into its own .lbug file.""" + """Fetch + ingest one repo into its own .lbug file (text pass, then structure).""" db_path.parent.mkdir(parents=True, exist_ok=True) - if reset: + if args.reset: for p in sorted(db_path.parent.glob(db_path.name + "*")): logger.info("[%s] reset: removing %s", repo, p.name) p.unlink() - logger.info("[%s] fetching up to %d merged PRs", repo, limit) - prs = fetch_merged_prs(repo, limit, token) - logger.info("[%s] fetched %d merged PRs -> %s", repo, len(prs), db_path.name) + logger.info("[%s] fetching up to %d merged PRs", repo, args.limit) + prs = fetch_merged_prs(repo, args.limit, token) + issues = fetch_issues(repo, args.issues, token) + commits = fetch_commits(repo, args.commits, token) + if args.files: + attach_pr_files(repo, prs, token) + logger.info( + "[%s] fetched %d PRs, %d issues, %d commits -> %s", + repo, len(prs), len(issues), len(commits), db_path.name, + ) db = TraceDB(db_path) db.init_schema() engine = CurationEngine(db) totals, skipped = IngestStats(), 0 try: - for pr in tqdm(prs, total=len(prs), desc=f"{repo}", unit="pr"): - doc_id, text = assemble_text(pr) - if not text.strip(): - skipped += 1 - continue - totals.merge( - ingest_text(engine, extractor, doc_id, text, source=pr.get("html_url")) - ) + if not args.no_text: + for pr in tqdm(prs, total=len(prs), desc=f"{repo}", unit="pr"): + doc_id, text = assemble_text(pr) + if not text.strip(): + skipped += 1 + continue + totals.merge( + ingest_text(engine, extractor, doc_id, text, + source=pr.get("html_url")) + ) + + # structured pass: relations read from the payload, not inferred. Runs + # after the text pass so a CO_OCCURS guess gets upgraded, not the reverse. + builder = GitHubGraphBuilder(db, engine.embed, repo) + graph = builder.build(pulls=prs, issues=issues, commits=commits) + db.build_vector_index() logger.info( "[%s] done. %d PRs (%d skipped) | %d entities | " @@ -150,6 +220,14 @@ def ingest_repo( totals.ollama_calls, totals.relates_edges, totals.mentions_edges, db.count_nodes(), ) + logger.info( + "[%s] structure: %d nodes, %d typed edges %s", + repo, graph.nodes, graph.edges, + dict(sorted(graph.by_relation.items(), key=lambda kv: -kv[1])), + ) + if not args.files: + logger.info("[%s] no TOUCHES edges from files — rerun with --files " + "to link PRs to the code they changed.", repo) finally: db.close() @@ -176,14 +254,15 @@ def main(argv: list[str] | None = None) -> int: if args.db and len(args.repo) > 1: logger.warning("--db is ignored with multiple repos; using per-repo files.") - extractor = EntityExtractor() + # --no-text skips extraction entirely; don't pay to load the models + extractor = None if args.no_text else EntityExtractor() for repo in args.repo: if args.db and len(args.repo) == 1: db_path = args.db if args.db.is_absolute() else (repo_root / args.db) else: db_path = repo_db_path(repo, graphs_dir) try: - ingest_repo(repo, db_path, args.limit, args.reset, token, extractor) + ingest_repo(repo, db_path, args, token, extractor) except Exception as exc: # noqa: BLE001 logger.error("[%s] FAILED: %s", repo, exc) diff --git a/backend/tests/test_db_integration.py b/backend/tests/test_db_integration.py new file mode 100644 index 0000000..7da008d --- /dev/null +++ b/backend/tests/test_db_integration.py @@ -0,0 +1,279 @@ +"""End-to-end against a real .lbug store. + +The unit tests exercise the scoring maths against a fake store; these run the +actual DDL, MERGE ... ON MATCH clauses and vector index through LadybugDB, so a +Cypher typo or a bad migration fails here instead of in production. + +Skipped when the driver isn't installed: pip install ladybug +""" +import shutil +import time + +import pytest + +lb = pytest.importorskip("ladybug", reason="LadybugDB driver not installed") + +from tracerag import config # noqa: E402 +from tracerag.db import TraceDB # noqa: E402 +from tracerag.github_graph import GitHubGraphBuilder # noqa: E402 +from tracerag.router import TraceRouter # noqa: E402 + +DAY = 86400 +NOW = int(time.time()) + + +def vec(*weights: float) -> list[float]: + """Unit-ish embedding: the given weights, zero-padded to EMBED_DIM.""" + v = list(weights) + [0.0] * (config.EMBED_DIM - len(weights)) + return [float(x) for x in v] + + +QUERY_VEC = vec(1.0) +NEAR = vec(0.93, 0.37) # cosine ~0.93 to QUERY_VEC +FAR = vec(0.45, 0.89) # cosine ~0.45 + + +@pytest.fixture +def db(tmp_path): + store = tmp_path / "t.lbug" + d = TraceDB(store, pool_size=2) + d.init_schema() + yield d + d.close() + shutil.rmtree(store, ignore_errors=True) + + +def add(d: TraceDB, node_id, label, ntype, embedding=None, ts=0): + d.upsert_node(node_id, label, ntype, embedding or vec(0.5, 0.5), ts=ts) + return node_id + + +# --- schema --------------------------------------------------------------- + +def test_schema_has_the_new_columns(db): + assert "ts" in db.table_columns(config.NODE_TABLE) + assert {"relation", "ts"} <= db.table_columns(config.REL_TABLE) + + +def test_fresh_store_needs_no_migration(db): + assert db.migrate_schema() == [] + + +def test_pre_03_store_migrates_in_place(tmp_path): + """An old .lbug must keep its data and gain the columns — not be deleted.""" + store = tmp_path / "old.lbug" + raw = lb.Database(str(store)) + conn = lb.Connection(raw) + conn.execute( + f"CREATE NODE TABLE {config.NODE_TABLE} (id STRING PRIMARY KEY, " + f"label STRING, type STRING, embedding FLOAT[{config.EMBED_DIM}]);" + ) + conn.execute( + f"CREATE REL TABLE {config.REL_TABLE} " + f"(FROM {config.NODE_TABLE} TO {config.NODE_TABLE}, confidence DOUBLE);" + ) + conn.execute( + f"CREATE (:{config.NODE_TABLE} {{id:'legacy', label:'Legacy', " + f"type:'PR', embedding:{QUERY_VEC}}});" + ) + conn.execute( + f"CREATE (:{config.NODE_TABLE} {{id:'legacy2', label:'Legacy2', " + f"type:'PR', embedding:{FAR}}});" + ) + # the old writer stored every edge at a degenerate 1.0 + conn.execute( + f"MATCH (a:{config.NODE_TABLE}{{id:'legacy'}}), " + f"(b:{config.NODE_TABLE}{{id:'legacy2'}}) " + f"CREATE (a)-[:{config.REL_TABLE} {{confidence: 1.0}}]->(b);" + ) + conn.close() + raw.close() + + d = TraceDB(store, pool_size=1) + added = d.migrate_schema() + assert set(added) == { + f"{config.NODE_TABLE}.ts", + f"{config.REL_TABLE}.relation", + f"{config.REL_TABLE}.ts", + } + # the pre-existing rows survived and read back with the column default + rows = d._fetch( + f"MATCH (e:{config.NODE_TABLE}) RETURN e.id AS id, e.ts AS ts " + f"ORDER BY e.id;") + assert rows == [{"id": "legacy", "ts": 0}, {"id": "legacy2", "ts": 0}] + assert edge(d)["relation"] == config.RELATION_CO_OCCURS + + # the degenerate 1.0 is repriced, so a typed relation can now outrank it + legacy_conf = edge(d)["confidence"] + assert legacy_conf == pytest.approx( + config.RELATION_WEIGHTS[config.RELATION_CO_OCCURS]) + assert legacy_conf < config.RELATION_WEIGHTS[config.RELATION_AUTHORED] + + # ...and upgrade it, which a 1.0 edge could never have allowed + d.add_relationship("legacy", "legacy2", + relation=config.RELATION_AUTHORED, ts=NOW) + assert edge(d)["relation"] == config.RELATION_AUTHORED + + # the migrated store accepts a fresh typed, timestamped write + add(d, "new", "New", "PR", NEAR, ts=NOW) + d.add_relationship("legacy", "new", relation=config.RELATION_RESOLVES, ts=NOW) + assert d.migrate_schema() == [] # idempotent + d.close() + + +# --- nodes ---------------------------------------------------------------- + +def test_node_timestamp_only_moves_forward(db): + add(db, "n", "N", "PR", ts=NOW - 10 * DAY) + add(db, "n", "N", "PR", ts=NOW) # newer event wins + add(db, "n", "N", "PR", ts=NOW - 90 * DAY) # older event must not roll it back + rows = db._fetch( + f"MATCH (e:{config.NODE_TABLE} {{id:'n'}}) RETURN e.ts AS ts;") + assert rows[0]["ts"] == NOW + + +def test_undated_rewrite_over_a_dated_row_does_not_overflow(db): + """Regression: $ts=0 was inferred as INT8, so the CASE overflowed against a + real epoch value. Re-ingesting an undated item must just be a no-op.""" + add(db, "n", "N", "PR", ts=NOW) + add(db, "n", "N", "PR", ts=0) + add(db, "m", "M", "PR") + db.add_relationship("n", "m", relation=config.RELATION_AUTHORED, ts=NOW) + db.add_relationship("n", "m", relation=config.RELATION_AUTHORED, ts=0) + node = db._fetch( + f"MATCH (e:{config.NODE_TABLE} {{id:'n'}}) RETURN e.ts AS ts;") + assert node[0]["ts"] == NOW + assert edge(db)["ts"] == NOW + + +# --- edges ---------------------------------------------------------------- + +def edge(db) -> dict: + rows = db._fetch( + f"MATCH ()-[r:{config.REL_TABLE}]->() " + f"RETURN r.confidence AS confidence, r.relation AS relation, r.ts AS ts;") + return rows[0] + + +def test_relation_defaults_to_its_configured_weight(db): + add(db, "a", "A", "Person") + add(db, "b", "B", "PR") + db.add_relationship("a", "b", relation=config.RELATION_AUTHORED, ts=NOW) + e = edge(db) + assert e["relation"] == config.RELATION_AUTHORED + assert e["confidence"] == pytest.approx( + config.RELATION_WEIGHTS[config.RELATION_AUTHORED]) + assert e["ts"] == NOW + + +def test_proximity_edge_upgrades_when_structure_proves_it(db): + add(db, "a", "A", "Person") + add(db, "b", "B", "PR") + db.add_relationship("a", "b", relation=config.RELATION_CO_OCCURS) + db.add_relationship("a", "b", relation=config.RELATION_AUTHORED, ts=NOW) + e = edge(db) + assert e["relation"] == config.RELATION_AUTHORED + assert e["confidence"] == pytest.approx( + config.RELATION_WEIGHTS[config.RELATION_AUTHORED]) + + +def test_a_later_guess_never_downgrades_a_known_edge(db): + add(db, "a", "A", "Person") + add(db, "b", "B", "PR") + db.add_relationship("a", "b", relation=config.RELATION_AUTHORED, ts=NOW) + db.add_relationship("a", "b", relation=config.RELATION_CO_OCCURS) + assert edge(db)["relation"] == config.RELATION_AUTHORED + + +def test_expand_frontier_carries_relation_and_ts(db): + add(db, "a", "A", "Person") + add(db, "b", "B", "PR", ts=NOW - 3 * DAY) + db.add_relationship("a", "b", relation=config.RELATION_AUTHORED, ts=NOW) + nbrs = db.expand_frontier(["a"], k=5, max_degree=10)["a"] + assert nbrs[0]["relation"] == config.RELATION_AUTHORED + assert nbrs[0]["ts"] == NOW - 3 * DAY + + +def test_subgraph_carries_relation_for_the_canvas(db): + add(db, "a", "A", "Person") + add(db, "b", "B", "PR") + db.add_relationship("a", "b", relation=config.RELATION_AUTHORED, ts=NOW) + sub = db.subgraph(["a"]) + assert sub["edges"][0]["relation"] == config.RELATION_AUTHORED + + +def test_vector_search_carries_ts(db): + add(db, "a", "A", "PR", NEAR, ts=NOW - 5 * DAY) + db.build_vector_index() + hit = db.vector_search(QUERY_VEC, k=1)[0] + assert hit["id"] == "a" + assert hit["ts"] == NOW - 5 * DAY + + +# --- github builder against the real store -------------------------------- + +PULL = { + "number": 412, "title": "Fix token refresh", "merged_at": "2026-07-27T10:00:00Z", + "user": {"login": "vishal"}, + "requested_reviewers": [{"login": "arush"}], + "body": "Fixes #77", + "files": [{"filename": "auth/session.py"}], +} + + +def test_builder_writes_typed_edges_into_a_real_store(db): + b = GitHubGraphBuilder(db, lambda text: vec(0.5, 0.5), "acme/api") + stats = b.build(pulls=[PULL]) + rows = db._fetch( + f"MATCH (a:{config.NODE_TABLE})-[r:{config.REL_TABLE}]->" + f"(c:{config.NODE_TABLE}) " + f"RETURN a.id AS src, c.id AS dst, r.relation AS relation;") + seen = {(r["src"], r["dst"], r["relation"]) for r in rows} + assert ("person:vishal", "pr:acme/api#412", + config.RELATION_AUTHORED) in seen + assert ("person:arush", "pr:acme/api#412", + config.RELATION_REVIEWED) in seen + assert ("pr:acme/api#412", "issue:acme/api#77", + config.RELATION_RESOLVES) in seen + assert ("pr:acme/api#412", "file:acme/api:auth/session.py", + config.RELATION_TOUCHES) in seen + assert stats.edges == len(rows) + + +# --- the whole thing ------------------------------------------------------ + +def test_stale_but_similar_loses_to_fresh_and_connected(db): + """The failure demo, run through the real DB rather than a fake one.""" + stale_ts = NOW - 240 * DAY + fresh_ts = NOW - 1 * DAY + add(db, "pr:stale", "PR #101", "PR", NEAR, ts=stale_ts) + add(db, "pr:fresh", "PR #412", "PR", FAR, ts=fresh_ts) + add(db, "person:vishal", "vishal", "Person", vec(0.4, 0.4, 0.4)) + db.add_relationship("person:vishal", "pr:fresh", + relation=config.RELATION_AUTHORED, ts=fresh_ts) + db.add_relationship("pr:stale", "person:vishal", + relation=config.RELATION_CO_OCCURS) + db.build_vector_index() + + router = TraceRouter(db) + router._encode = lambda text: QUERY_VEC # skip the real embedder + + # vector-only sanity: the stale PR genuinely looks more similar + hits = {h["id"]: h["similarity"] for h in db.vector_search(QUERY_VEC, k=5)} + assert hits["pr:stale"] > hits["pr:fresh"] + + res = router.route("who owns the code that broke?", top_k=10) + ranked = [r.id for r in res.results] + assert ranked.index("pr:fresh") < ranked.index("pr:stale"), ranked + + by_id = {r.id: r for r in res.results} + assert by_id["pr:stale"].recency < by_id["pr:fresh"].recency + assert by_id["pr:stale"].age_days == pytest.approx(240, abs=1) + + hops = res.trace_log["execution_path"]["graph_hops"] + assert {h["relation"] for h in hops} >= {config.RELATION_AUTHORED} + assert res.trace_log["recency"]["enabled"] is True + + +if __name__ == "__main__": + pytest.main([__file__, "-q"]) diff --git a/backend/tests/test_ingest_github_wiring.py b/backend/tests/test_ingest_github_wiring.py new file mode 100644 index 0000000..4dc8328 --- /dev/null +++ b/backend/tests/test_ingest_github_wiring.py @@ -0,0 +1,136 @@ +"""The live GitHub path actually reaches GitHubGraphBuilder. + +The builder has its own unit tests against fake payloads; these check the +wiring — that ingest_github fetches the extra endpoints, filters them right, +and lands typed edges in a real store. Network calls are stubbed. +""" +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("ladybug", reason="LadybugDB driver not installed") +pytest.importorskip("requests") +pytest.importorskip("tqdm") + +_BACKEND = Path(__file__).resolve().parents[1] +for p in (_BACKEND, _BACKEND / "scripts"): + if str(p) not in sys.path: + sys.path.insert(0, str(p)) + +import ingest_github as gh # noqa: E402 +from tracerag import config # noqa: E402 +from tracerag.db import TraceDB # noqa: E402 + +EMBED = [0.0] * config.EMBED_DIM +EMBED[0] = 1.0 + +PULL = { + "number": 412, "title": "Fix token refresh", "merged_at": "2026-07-27T10:00:00Z", + "html_url": "https://github.com/acme/api/pull/412", + "user": {"login": "vishal"}, + "requested_reviewers": [{"login": "arush"}], + "body": "Fixes #77", +} +ISSUE = {"number": 77, "title": "Sessions drop", "created_at": "2026-07-20T08:00:00Z", + "user": {"login": "utsav"}} +COMMIT = {"sha": "abc1234def", "author": {"login": "vishal"}, + "commit": {"message": "hotfix, closes #77", + "author": {"date": "2026-07-27T09:00:00Z"}}} +FILES = [{"filename": "auth/session.py"}] + + +# --- fetch filters -------------------------------------------------------- + +def test_only_merged_prs_are_kept(monkeypatch): + pages = [[{"number": 1, "merged_at": "2026-01-01T00:00:00Z"}, + {"number": 2, "merged_at": None}], []] + monkeypatch.setattr(gh, "_get", lambda *a, **k: pages.pop(0)) + assert [p["number"] for p in gh.fetch_merged_prs("acme/api", 10, None)] == [1] + + +def test_the_issues_endpoint_drops_pull_requests(monkeypatch): + pages = [[{"number": 5}, {"number": 6, "pull_request": {"url": "..."}}], []] + monkeypatch.setattr(gh, "_get", lambda *a, **k: pages.pop(0)) + assert [i["number"] for i in gh.fetch_issues("acme/api", 10, None)] == [5] + + +def test_zero_limits_skip_the_request_entirely(monkeypatch): + def boom(*a, **k): + raise AssertionError("should not have called the API") + + monkeypatch.setattr(gh, "_get", boom) + assert gh.fetch_issues("acme/api", 0, None) == [] + assert gh.fetch_commits("acme/api", 0, None) == [] + + +def test_a_failed_file_fetch_does_not_abort_the_run(monkeypatch): + def boom(*a, **k): + raise RuntimeError("GitHub API 403") + + monkeypatch.setattr(gh, "_get", boom) + prs = [dict(PULL)] + gh.attach_pr_files("acme/api", prs, None) # must not raise + assert "files" not in prs[0] + + +# --- the wiring ----------------------------------------------------------- + +@pytest.fixture +def ingested(tmp_path, monkeypatch): + monkeypatch.setattr(gh, "fetch_merged_prs", lambda *a, **k: [dict(PULL)]) + monkeypatch.setattr(gh, "fetch_issues", lambda *a, **k: [ISSUE]) + monkeypatch.setattr(gh, "fetch_commits", lambda *a, **k: [COMMIT]) + monkeypatch.setattr(gh, "attach_pr_files", + lambda repo, prs, token: [pr.update(files=FILES) for pr in prs]) + # stub the embedder — this test is about wiring, not vectors + monkeypatch.setattr(gh.CurationEngine, "embed", lambda self, text: list(EMBED)) + + db_path = tmp_path / "acme.lbug" + args = gh.parse_args(["--repo", "acme/api", "--no-text", + "--issues", "5", "--commits", "5", "--files"]) + gh.ingest_repo("acme/api", db_path, args, None, None) + + db = TraceDB(db_path, pool_size=1) + yield db + db.close() + + +def edges(db) -> set[tuple[str, str, str]]: + rows = db._fetch( + f"MATCH (a:{config.NODE_TABLE})-[r:{config.REL_TABLE}]->" + f"(b:{config.NODE_TABLE}) " + f"RETURN a.id AS src, b.id AS dst, r.relation AS relation;") + return {(r["src"], r["dst"], r["relation"]) for r in rows} + + +def test_live_path_writes_every_relation_kind(ingested): + seen = edges(ingested) + assert ("person:vishal", "pr:acme/api#412", config.RELATION_AUTHORED) in seen + assert ("person:arush", "pr:acme/api#412", config.RELATION_REVIEWED) in seen + assert ("pr:acme/api#412", "issue:acme/api#77", config.RELATION_RESOLVES) in seen + assert ("pr:acme/api#412", "file:acme/api:auth/session.py", + config.RELATION_TOUCHES) in seen + assert ("person:utsav", "issue:acme/api#77", config.RELATION_REPORTED) in seen + assert ("commit:acme/api:abc1234", "repo:acme/api", + config.RELATION_PART_OF) in seen + assert ("commit:acme/api:abc1234", "issue:acme/api#77", + config.RELATION_RESOLVES) in seen + + +def test_the_pr_is_dated_by_its_merge_not_by_now(ingested): + from tracerag.github_graph import parse_ts + + rows = ingested._fetch( + f"MATCH (e:{config.NODE_TABLE} {{id:'pr:acme/api#412'}}) RETURN e.ts AS ts;") + assert rows[0]["ts"] == parse_ts(PULL["merged_at"]) + + +def test_no_relation_is_left_untyped(ingested): + relations = {r for _, _, r in edges(ingested)} + assert config.RELATION_CO_OCCURS not in relations, ( + "--no-text ran, so every edge should come from structure") + + +if __name__ == "__main__": + pytest.main([__file__, "-q"]) diff --git a/backend/tracerag/curation.py b/backend/tracerag/curation.py index c20f43d..e99dcd2 100644 --- a/backend/tracerag/curation.py +++ b/backend/tracerag/curation.py @@ -69,6 +69,11 @@ def _embed(self, text: str) -> list[float]: vec = self._get_embedder().encode(text, normalize_embeddings=True) return [float(x) for x in vec] + def embed(self, text: str) -> list[float]: + """Public handle on the shared vector space. The GitHub graph builder + writes structured nodes through this, so both paths land in one index.""" + return self._embed(text) + def _get_llm(self): if self._llm is None: from .llm import make_client diff --git a/backend/tracerag/db.py b/backend/tracerag/db.py index f24091c..780e1cb 100644 --- a/backend/tracerag/db.py +++ b/backend/tracerag/db.py @@ -15,6 +15,25 @@ logger = logging.getLogger(__name__) +_TS = "CAST($ts AS INT64)" # see upsert_node — small ints infer as INT8 + + +def _added_columns() -> dict[str, list[tuple[str, str, str]]]: + """Columns that arrived after the first schema shipped: (name, type, default). + + CREATE TABLE IF NOT EXISTS is a no-op on a store that already exists, so a + pre-0.3 .lbug keeps the old shape and only fails later, at query time, on + e.ts / r.relation. ALTER ... ADD backfills existing rows with the default, + so migrating is lossless — no reingest. + """ + return { + config.NODE_TABLE: [("ts", "INT64", "0")], + config.REL_TABLE: [ + ("relation", "STRING", f"'{config.RELATION_CO_OCCURS}'"), + ("ts", "INT64", "0"), + ], + } + class TraceDB: """Wrapper over LadybugDB with a read-connection pool. @@ -128,10 +147,65 @@ def init_schema(self) -> None: f"CREATE REL TABLE IF NOT EXISTS {config.MENTIONS_TABLE} (" f"FROM {config.DOC_TABLE} TO {config.NODE_TABLE});" ) + self.migrate_schema() # HNSW index built later via build_vector_index(); a live index blocks embedding writes logger.info("Schema ready (node=%s, rel=%s)", config.NODE_TABLE, config.REL_TABLE) + def table_columns(self, table: str) -> set[str]: + """Column names on a table; empty set when the table doesn't exist.""" + try: + rows = self._fetch(f"CALL TABLE_INFO('{table}') RETURN *;") + except Exception as exc: # noqa: BLE001 — table absent, or older engine + logger.debug("TABLE_INFO('%s') unavailable: %s", table, exc) + return set() + return {r["name"] for r in rows} + + def migrate_schema(self) -> list[str]: + """Bring an older store up to the current shape. Returns what it added.""" + added: list[str] = [] + for table, columns in _added_columns().items(): + existing = self.table_columns(table) + if not existing: + continue # freshly created above, already correct + for name, sql_type, default in columns: + if name in existing: + continue + self.execute( + f"ALTER TABLE {table} ADD {name} {sql_type} DEFAULT {default};" + ) + added.append(f"{table}.{name}") + if (table, name) == (config.REL_TABLE, "relation"): + self._downgrade_legacy_confidence() + if added: + logger.warning( + "Migrated %s to the 0.3 schema: added %s. Existing rows use the " + "column default — reingest to populate real timestamps and relations.", + self.db_path, ", ".join(added), + ) + return added + + def _downgrade_legacy_confidence(self) -> None: + """Reprice pre-0.3 edges as what they actually are: proximity guesses. + + The old writer stored every edge at 1.0, which made traversal flat. Left + alone those edges would also outrank every typed relation (max 0.95) and + could never be upgraded, since the ON MATCH guard needs a strictly higher + confidence. The ALTER just labelled them all CO_OCCURS — so give them the + CO_OCCURS weight to match. + """ + weight = config.RELATION_WEIGHTS[config.RELATION_CO_OCCURS] + self.execute( + f"MATCH ()-[r:{config.REL_TABLE}]->() " + f"WHERE r.confidence > $weight SET r.confidence = $weight;", + {"weight": float(weight)}, + ) + logger.warning( + "Repriced legacy edges to the %s weight (%.2f) so structured " + "relations can outrank and upgrade them.", + config.RELATION_CO_OCCURS, weight, + ) + def build_vector_index(self) -> None: """(Re)build the HNSW index over current embeddings; call after ingestion.""" self._load_vector_extension() @@ -163,7 +237,9 @@ def upsert_node( f"MERGE (e:{config.NODE_TABLE} {{id: $id}}) " f"ON CREATE SET e.label = $label, e.type = $type, e.ts = $ts, " f"e.embedding = $embedding " - f"ON MATCH SET e.ts = CASE WHEN $ts > e.ts THEN $ts ELSE e.ts END", + # CAST is load-bearing: the driver infers a small $ts (0 for undated + # items) as INT8, and the CASE then overflows on a real epoch value. + f"ON MATCH SET e.ts = CASE WHEN {_TS} > e.ts THEN {_TS} ELSE e.ts END", {"id": node_id, "label": label, "type": node_type, "ts": int(ts), "embedding": list(embedding)}, ) @@ -187,11 +263,15 @@ def add_relationship( f"(b:{config.NODE_TABLE} {{id: $to_id}}) " f"MERGE (a)-[r:{config.REL_TABLE}]->(b) " f"ON CREATE SET r.confidence = $confidence, r.relation = $relation, r.ts = $ts " - # a stronger relation later (CO_OCCURS then AUTHORED) upgrades the edge - f"ON MATCH SET r.confidence = CASE WHEN $confidence > r.confidence " - f"THEN $confidence ELSE r.confidence END, " - f"r.relation = CASE WHEN $confidence > r.confidence THEN $relation " - f"ELSE r.relation END", + # a stronger relation later (CO_OCCURS then AUTHORED) upgrades the edge. + # SET items apply left to right, so relation must be decided BEFORE + # confidence is overwritten — otherwise its guard compares the new + # value against itself and the label never moves. + f"ON MATCH SET r.relation = CASE WHEN $confidence > r.confidence " + f"THEN $relation ELSE r.relation END, " + f"r.ts = CASE WHEN {_TS} > r.ts THEN {_TS} ELSE r.ts END, " + f"r.confidence = CASE WHEN $confidence > r.confidence " + f"THEN $confidence ELSE r.confidence END", {"from_id": from_id, "to_id": to_id, "confidence": float(confidence), "relation": relation, "ts": int(ts)}, ) @@ -360,12 +440,14 @@ def _add(nid, label, ntype): f"MATCH (a:{config.NODE_TABLE})-[r:{config.REL_TABLE}]->" f"(b:{config.NODE_TABLE}) " f"WHERE a.id IN $ids AND b.id IN $ids " - f"RETURN a.id AS source, b.id AS target, r.confidence AS confidence;", + f"RETURN a.id AS source, b.id AS target, r.confidence AS confidence, " + f"r.relation AS relation;", {"ids": all_ids}, ) if all_ids else [] edges = [ {"source": r["source"], "target": r["target"], - "confidence": r["confidence"]} + "confidence": r["confidence"], + "relation": r.get("relation") or config.RELATION_CO_OCCURS} for r in edge_rows ] return {"nodes": list(nodes.values()), "edges": edges} diff --git a/frontend/src/components/right/EntityNode.tsx b/frontend/src/components/right/EntityNode.tsx index 4edb6ce..bdc2560 100644 --- a/frontend/src/components/right/EntityNode.tsx +++ b/frontend/src/components/right/EntityNode.tsx @@ -16,11 +16,24 @@ import type { TraceNode } from "@/types/trace"; export type EntityNodeData = TraceNode; +// compact age for the node chip: 4d, 3w, 8mo, 2y +function formatAge(days: number): string { + if (days < 1) return "today"; + if (days < 14) return `${Math.round(days)}d`; + if (days < 60) return `${Math.round(days / 7)}w`; + if (days < 365) return `${Math.round(days / 30)}mo`; + return `${(days / 365).toFixed(1)}y`; +} + function EntityNodeComponent({ data, selected }: NodeProps) { const style = ENTITY_STYLES[data.type]; const Icon = style.icon; const active = data.active; const snippet = data.meta?.snippet; + const ageDays = data.meta?.ageDays; + const recency = data.meta?.recency; + // only surface age once decay actually moved the score — below that it's noise + const decayed = recency != null && recency < 0.95 && ageDays != null; // fetched lazily when the hover card opens (server-cached) const [summary, setSummary] = useState(null); @@ -78,6 +91,18 @@ function EntityNodeComponent({ data, selected }: NodeProps) {

{style.label} + {decayed && ( + + {formatAge(ageDays!)} + + )}

@@ -129,6 +154,7 @@ function EntityNodeComponent({ data, selected }: NodeProps) { {(data.score != null || data.similarity != null || (data.meta?.scoreGraph ?? 0) > 0 || + recency != null || data.meta?.connections != null) && ( <> @@ -146,6 +172,17 @@ function EntityNodeComponent({ data, selected }: NodeProps) { mono /> )} + {recency != null && ( + + )} {data.meta?.connections != null && ( ; } @@ -47,6 +56,8 @@ export interface TraceResult { score_total?: number; score_vector?: number; score_graph?: number; + recency?: number; + age_days?: number | null; documents?: { doc_id?: string; content?: string; path?: string }[]; page_content?: string; } @@ -347,6 +358,24 @@ function buildSteps(log: TraceLog): ExecutionStep[] { const seeds = log.execution_path.vector_seeds.length; const hops = log.execution_path.graph_hops.length; const evaluated = log.metrics.total_nodes_evaluated; + + // name the relations actually walked — "3 edges" says nothing about why + const byRelation = new Map(); + for (const h of log.execution_path.graph_hops) { + if (h.relation) byRelation.set(h.relation, (byRelation.get(h.relation) ?? 0) + 1); + } + const relations = [...byRelation.entries()] + .sort((a, b) => b[1] - a[1]) + .map(([rel, n]) => `${n}× ${rel}`) + .join(", "); + + // recency only matters when it actually moved something + const decayed = (log.recency?.applied ?? []).filter((a) => a.factor < 0.95); + const oldest = decayed.reduce( + (worst, a) => ((a.age_days ?? 0) > (worst?.age_days ?? 0) ? a : worst), + decayed[0] + ); + return [ { id: "step-1", @@ -361,7 +390,9 @@ function buildSteps(log: TraceLog): ExecutionStep[] { id: "step-2", index: 2, title: "Graph Traversal", - detail: `Walked ${hops} edge${hops === 1 ? "" : "s"} across the relationship graph.`, + detail: relations + ? `Walked ${hops} edge${hops === 1 ? "" : "s"}: ${relations}.` + : `Walked ${hops} edge${hops === 1 ? "" : "s"} across the relationship graph.`, status: "complete", badge: `${hops} hop${hops === 1 ? "" : "s"}`, arm: "graph", @@ -370,9 +401,15 @@ function buildSteps(log: TraceLog): ExecutionStep[] { id: "step-3", index: 3, title: "Context Assembly", - detail: `Evaluated ${evaluated} node${evaluated === 1 ? "" : "s"} to assemble the grounded context.`, + detail: + decayed.length > 0 + ? `Evaluated ${evaluated} node${evaluated === 1 ? "" : "s"}; age decay demoted ${ + decayed.length + } of them (oldest ${Math.round(oldest.age_days ?? 0)} days, ×${oldest.factor.toFixed(2)}).` + : `Evaluated ${evaluated} node${evaluated === 1 ? "" : "s"} to assemble the grounded context.`, status: "complete", - badge: `${evaluated} nodes`, + badge: + decayed.length > 0 ? `${decayed.length} aged` : `${evaluated} nodes`, }, ]; } @@ -421,6 +458,8 @@ export function adaptToTraceState( snippet: r ? resultSnippet(r) : undefined, scoreGraph: r?.score_graph, sourceUrl: r ? resultSourceUrl(r) : undefined, + recency: r?.recency, + ageDays: r?.age_days ?? null, }, }); } @@ -442,6 +481,8 @@ export function adaptToTraceState( snippet: r ? resultSnippet(r) : undefined, scoreGraph: r?.score_graph, sourceUrl: r ? resultSourceUrl(r) : undefined, + recency: r?.recency, + ageDays: r?.age_days ?? null, }, }); } @@ -479,6 +520,7 @@ export function adaptToTraceState( target: h.to_id, confidence: h.confidence, active: true, + relation: h.relation, }); } }); diff --git a/frontend/src/types/trace.ts b/frontend/src/types/trace.ts index c5231d3..e154fd7 100644 --- a/frontend/src/types/trace.ts +++ b/frontend/src/types/trace.ts @@ -32,6 +32,10 @@ export interface TraceNode { connections?: number; scoreGraph?: number; sourceUrl?: string; + // age decay applied to the fused score: 1 = untouched, lower = older + recency?: number; + // null when the entity carries no timestamp, so it was never decayed + ageDays?: number | null; }; // disconnected nodes hidden behind the "Show Orphans" toggle orphan?: boolean;