Graphsight

.github/workflows/ci.yml

@@ -25,6 +25,29 @@ jobs:
       - name: run tests
         run: pytest graphsight-langgraph/tests graphsight/tests -q
 
+  # The graph engine: scoring maths plus the real LadybugDB path (schema,
+  # migration, MERGE semantics). Deliberately avoids torch/spacy — these tests
+  # stub the embedder, so the whole job installs in seconds.
+  engine-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: install engine test deps
+        run: pip install pytest ladybug numpy pandas requests tqdm
+      # named explicitly: the rest of backend/tests needs the full API stack
+      - name: run engine tests
+        run: >
+          pytest -q
+          tests/test_recency.py
+          tests/test_router_scoring.py
+          tests/test_github_graph.py
+          tests/test_db_integration.py
+          tests/test_ingest_github_wiring.py
+        working-directory: backend
+
   frontend-build:
     runs-on: ubuntu-latest
     steps:

README.md

@@ -285,10 +285,25 @@ hf-space/                   # HuggingFace Space deployment (own git remote)
 
 ### Data model
 
-A single generic `Entity` node table (`id, label, type, embedding`) keeps the schema dynamic
+A single generic `Entity` node table (`id, label, type, ts, embedding`) keeps the schema dynamic
 across all entity labels. `Document` nodes store one **sliding-window chunk** each (with raw
 `content` for generation). Edges: `Document -[MENTIONS]-> Entity` and `Entity -[RELATES_TO]->
-Entity` (only between entities co-occurring in the *same* window — no document-wide hairball).
+Entity`, carrying `confidence, relation, ts`.
+
+`relation` is one of `AUTHORED · RESOLVES · REVIEWED · TOUCHES · PART_OF · REPORTED ·
+CO_OCCURS`, each with its own weight (`config.RELATION_WEIGHTS`). Text ingestion can only
+produce `CO_OCCURS` — same-window proximity, weighted lowest. Structured ingestion reads the
+rest from the source API, so a two-hop `AUTHORED → RESOLVES` chain outranks a one-hop guess.
+An edge **upgrades** if structure later proves a guess right; it never downgrades.
+
+`ts` is the underlying event time (0 when unknown), which drives recency decay:
+`score *= 0.5 ** (age_days / half_life)`, floored at `RECENCY_FLOOR`. Undated entities are
+left alone rather than penalised.
+
+**Opening an older store migrates it in place.** `init_schema()` adds the missing columns via
+`ALTER TABLE ... ADD ... DEFAULT`, keeping every existing row, and reprices legacy edges —
+the pre-0.3 writer stored them all at a degenerate `1.0`, which both flattened traversal and
+would have outranked every typed relation. Reingest to populate real timestamps and relations.
 
 ### High-Level Design (HLD)
 
@@ -927,6 +942,13 @@ python -m spacy download en_core_web_sm        # fallback NER
 # 2. ingest your datasets/ (writes memory.lbug, builds the HNSW index)
 python scripts/ingest.py --datasets ./datasets --reset
 
+# 2b. or ingest a GitHub repo. Two passes land in one graph: PR prose through
+#     extraction, and the same payloads through GitHubGraphBuilder, which reads
+#     authorship, reviews, "Fixes #N" and touched files as typed, dated edges.
+python scripts/ingest_github.py --repo pallets/click --issues 50 --commits 50 --files
+#     --files costs one extra API request per PR; without it there are no
+#     TOUCHES edges. --no-text skips extraction and writes structure only.
+
 # 3. evaluate (optional)
 python scripts/benchmark.py            # -> results.csv + summary table
 

backend/scripts/ingest_github.py

@@ -2,6 +2,14 @@
 
     python scripts/ingest_github.py --repo langchain-ai/langchain
     python scripts/ingest_github.py --repo owner/repo --limit 100 --reset
+    python scripts/ingest_github.py --repo owner/repo --issues 50 --commits 50 --files
+
+Two passes write into the same graph:
+
+  * text     — PR prose through extraction + curation (fuzzy CO_OCCURS edges)
+  * structure — the same payloads through GitHubGraphBuilder, which reads
+    authorship, reviews, "Fixes #N" and touched files as typed, timestamped
+    edges. These are facts from the API, not guesses from word proximity.
 
 Set GITHUB_TOKEN in .env to raise the rate limit from 60/hr to 5000/hr.
 """
@@ -26,6 +34,7 @@
 from tracerag.db import TraceDB                    # noqa: E402
 from tracerag.extract import EntityExtractor       # noqa: E402
 from tracerag.curation import CurationEngine, IngestStats  # noqa: E402
+from tracerag.github_graph import GitHubGraphBuilder       # noqa: E402
 from ingest import ingest_text                     # noqa: E402
 
 logger = logging.getLogger("tracerag.github")
@@ -59,34 +68,71 @@ def assemble_text(pr: dict) -> tuple[str, str]:
     return f"pr-{number}", text
 
 
-def fetch_merged_prs(repo: str, limit: int, token: str | None) -> list[dict]:
-    """Page through closed PRs, keeping only merged ones (merged_at != null), until limit."""
-    headers = {
+def _headers(token: str | None) -> dict[str, str]:
+    h = {
         "Accept": "application/vnd.github+json",
         "X-GitHub-Api-Version": "2022-11-28",
     }
     if token:
-        headers["Authorization"] = f"Bearer {token}"
+        h["Authorization"] = f"Bearer {token}"
+    return h
+
+
+def _get(url: str, token: str | None, params: dict | None = None):
+    resp = requests.get(url, headers=_headers(token), params=params, timeout=30)
+    if resp.status_code != 200:
+        raise RuntimeError(f"GitHub API {resp.status_code} for {url}: {resp.text[:200]}")
+    return resp.json()
+
 
-    merged: list[dict] = []
+def _paged(url: str, limit: int, token: str | None, params: dict,
+           keep=lambda item: True) -> list[dict]:
+    """Page an endpoint until `limit` kept items, or the listing runs out."""
+    out: list[dict] = []
     page = 1
-    while len(merged) < limit:
-        resp = requests.get(
-            f"{GITHUB_API}/repos/{repo}/pulls",
-            headers=headers,
-            params={"state": "closed", "per_page": _PER_PAGE, "page": page},
-            timeout=30,
-        )
-        if resp.status_code != 200:
-            raise RuntimeError(
-                f"GitHub API {resp.status_code} for {repo}: {resp.text[:200]}"
-            )
-        batch = resp.json()
+    while len(out) < limit:
+        batch = _get(url, token, {**params, "per_page": _PER_PAGE, "page": page})
         if not batch:
             break
-        merged.extend(pr for pr in batch if pr.get("merged_at"))
+        out.extend(item for item in batch if keep(item))
         page += 1
-    return merged[:limit]
+    return out[:limit]
+
+
+def fetch_merged_prs(repo: str, limit: int, token: str | None) -> list[dict]:
+    """Page through closed PRs, keeping only merged ones (merged_at != null), until limit."""
+    return _paged(
+        f"{GITHUB_API}/repos/{repo}/pulls", limit, token,
+        {"state": "closed"}, keep=lambda pr: bool(pr.get("merged_at")),
+    )
+
+
+def fetch_issues(repo: str, limit: int, token: str | None) -> list[dict]:
+    """Issues only — the endpoint also returns PRs, which the builder skips."""
+    if limit <= 0:
+        return []
+    return _paged(
+        f"{GITHUB_API}/repos/{repo}/issues", limit, token,
+        {"state": "all"}, keep=lambda i: "pull_request" not in i,
+    )
+
+
+def fetch_commits(repo: str, limit: int, token: str | None) -> list[dict]:
+    if limit <= 0:
+        return []
+    return _paged(f"{GITHUB_API}/repos/{repo}/commits", limit, token, {})
+
+
+def attach_pr_files(repo: str, prs: list[dict], token: str | None) -> None:
+    """Fill each PR's `files` in place — one extra request per PR, so opt-in."""
+    for pr in tqdm(prs, desc=f"{repo} files", unit="pr", leave=False):
+        try:
+            pr["files"] = _get(
+                f"{GITHUB_API}/repos/{repo}/pulls/{pr['number']}/files",
+                token, {"per_page": _PER_PAGE},
+            )
+        except Exception as exc:  # noqa: BLE001 — a missing file list isn't fatal
+            logger.debug("[%s] files for #%s failed: %s", repo, pr.get("number"), exc)
 
 
 def repo_db_path(repo: str, graphs_dir: Path) -> Path:
@@ -101,6 +147,15 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
                    help='One or more repos, e.g. "pallets/flask psf/requests".')
     p.add_argument("--limit", type=int, default=50,
                    help="Max MERGED PRs to ingest per repo (protects rate limits).")
+    p.add_argument("--issues", type=int, default=0,
+                   help="Also ingest up to N issues (REPORTED edges, RESOLVES targets).")
+    p.add_argument("--commits", type=int, default=0,
+                   help="Also ingest up to N commits (AUTHORED + 'Fixes #N' edges).")
+    p.add_argument("--files", action="store_true",
+                   help="Fetch each PR's changed files for TOUCHES edges. "
+                        "Costs one extra API request per PR.")
+    p.add_argument("--no-text", action="store_true",
+                   help="Skip the prose/curation pass; write structured edges only.")
     p.add_argument("--db", type=Path, default=None,
                    help="Override output file (single-repo only; otherwise one "
                         "per-repo file is created under --graphs-dir).")
@@ -113,33 +168,48 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
 
 
 def ingest_repo(
-    repo: str, db_path: Path, limit: int, reset: bool,
-    token: str | None, extractor: EntityExtractor,
+    repo: str, db_path: Path, args: argparse.Namespace,
+    token: str | None, extractor: EntityExtractor | None,
 ) -> None:
-    """Fetch + ingest one repo's merged PRs into its own .lbug file."""
+    """Fetch + ingest one repo into its own .lbug file (text pass, then structure)."""
     db_path.parent.mkdir(parents=True, exist_ok=True)
-    if reset:
+    if args.reset:
         for p in sorted(db_path.parent.glob(db_path.name + "*")):
             logger.info("[%s] reset: removing %s", repo, p.name)
             p.unlink()
 
-    logger.info("[%s] fetching up to %d merged PRs", repo, limit)
-    prs = fetch_merged_prs(repo, limit, token)
-    logger.info("[%s] fetched %d merged PRs -> %s", repo, len(prs), db_path.name)
+    logger.info("[%s] fetching up to %d merged PRs", repo, args.limit)
+    prs = fetch_merged_prs(repo, args.limit, token)
+    issues = fetch_issues(repo, args.issues, token)
+    commits = fetch_commits(repo, args.commits, token)
+    if args.files:
+        attach_pr_files(repo, prs, token)
+    logger.info(
+        "[%s] fetched %d PRs, %d issues, %d commits -> %s",
+        repo, len(prs), len(issues), len(commits), db_path.name,
+    )
 
     db = TraceDB(db_path)
     db.init_schema()
     engine = CurationEngine(db)
     totals, skipped = IngestStats(), 0
     try:
-        for pr in tqdm(prs, total=len(prs), desc=f"{repo}", unit="pr"):
-            doc_id, text = assemble_text(pr)
-            if not text.strip():
-                skipped += 1
-                continue
-            totals.merge(
-                ingest_text(engine, extractor, doc_id, text, source=pr.get("html_url"))
-            )
+        if not args.no_text:
+            for pr in tqdm(prs, total=len(prs), desc=f"{repo}", unit="pr"):
+                doc_id, text = assemble_text(pr)
+                if not text.strip():
+                    skipped += 1
+                    continue
+                totals.merge(
+                    ingest_text(engine, extractor, doc_id, text,
+                                source=pr.get("html_url"))
+                )
+
+        # structured pass: relations read from the payload, not inferred. Runs
+        # after the text pass so a CO_OCCURS guess gets upgraded, not the reverse.
+        builder = GitHubGraphBuilder(db, engine.embed, repo)
+        graph = builder.build(pulls=prs, issues=issues, commits=commits)
+
         db.build_vector_index()
         logger.info(
             "[%s] done. %d PRs (%d skipped) | %d entities | "
@@ -150,6 +220,14 @@ def ingest_repo(
             totals.ollama_calls, totals.relates_edges, totals.mentions_edges,
             db.count_nodes(),
         )
+        logger.info(
+            "[%s] structure: %d nodes, %d typed edges %s",
+            repo, graph.nodes, graph.edges,
+            dict(sorted(graph.by_relation.items(), key=lambda kv: -kv[1])),
+        )
+        if not args.files:
+            logger.info("[%s] no TOUCHES edges from files — rerun with --files "
+                        "to link PRs to the code they changed.", repo)
     finally:
         db.close()
 
@@ -176,14 +254,15 @@ def main(argv: list[str] | None = None) -> int:
     if args.db and len(args.repo) > 1:
         logger.warning("--db is ignored with multiple repos; using per-repo files.")
 
-    extractor = EntityExtractor()
+    # --no-text skips extraction entirely; don't pay to load the models
+    extractor = None if args.no_text else EntityExtractor()
     for repo in args.repo:
         if args.db and len(args.repo) == 1:
             db_path = args.db if args.db.is_absolute() else (repo_root / args.db)
         else:
             db_path = repo_db_path(repo, graphs_dir)
         try:
-            ingest_repo(repo, db_path, args.limit, args.reset, token, extractor)
+            ingest_repo(repo, db_path, args, token, extractor)
         except Exception as exc:  # noqa: BLE001
             logger.error("[%s] FAILED: %s", repo, exc)