Graphsight

.gitignore

@@ -68,3 +68,4 @@ yarn-error.log*
 docs/DEMO_SCRIPT.md
 docs/STRESS_RESULTS.md
 docs/DEPLOY.md
+dist_release/

README.md

@@ -153,7 +153,7 @@ flowchart TB
     SEEDL --> SD["seed set · dedup"]
     SEEDF --> SD
     SD --> EF["expand_frontier<br/>ONE query per hop · no N+1"]
-    EF --> HUB["hub throttle<br/>skip degree above MAX_DEGREE"]
+    EF --> HUB["hub throttle<br/>drop a relation fanning out<br/>past MAX_DEGREE from one node"]
     HUB --> PS["path_score =<br/>product of edge confidences"]
     PS --> HOPQ{"hop below MAX_HOPS<br/>and frontier left?"}
     HOPQ -->|yes| EF
@@ -291,7 +291,10 @@ across all entity labels. `Document` nodes store one **sliding-window chunk** ea
 Entity`, carrying `confidence, relation, ts`.
 
 `relation` is one of `AUTHORED · RESOLVES · REVIEWED · TOUCHES · PART_OF · REPORTED ·
-CO_OCCURS`, each with its own weight (`config.RELATION_WEIGHTS`). Text ingestion can only
+CO_OCCURS`, each with its own weight (`config.RELATION_WEIGHTS`). `REVIEWED` comes from
+`/pulls/N/reviews` — **not** from `requested_reviewers`, which is a pending ask that empties
+once someone actually reviews, so reading it both misses real reviewers and credits people who
+never opened the code. Text ingestion can only
 produce `CO_OCCURS` — same-window proximity, weighted lowest. Structured ingestion reads the
 rest from the source API, so a two-hop `AUTHORED → RESOLVES` chain outranks a one-hop guess.
 An edge **upgrades** if structure later proves a guess right; it never downgrades.
@@ -416,7 +419,7 @@ flowchart TB
 
   subgraph BFS["graph_stream — multiplicative BFS, per hop"]
     direction TB
-    B1["expand_frontier(frontier)<br/>ONE query per hop (no N+1)"] --> B2["hub throttle:<br/>skip nodes with degree above MAX_DEGREE"]
+    B1["expand_frontier(frontier)<br/>ONE query per hop (no N+1)"] --> B2["hub throttle: drop a relation<br/>fanning out past MAX_DEGREE<br/>(repo TOUCHES everything);<br/>the node's other edges survive"]
     B2 --> B3["path_score = product of edge confidences<br/>keep max score per node"]
     B3 --> B4{"more hops left<br/>and frontier not empty?"}
     B4 -->|yes| B1
@@ -945,9 +948,11 @@ python scripts/ingest.py --datasets ./datasets --reset
 # 2b. or ingest a GitHub repo. Two passes land in one graph: PR prose through
 #     extraction, and the same payloads through GitHubGraphBuilder, which reads
 #     authorship, reviews, "Fixes #N" and touched files as typed, dated edges.
-python scripts/ingest_github.py --repo pallets/click --issues 50 --commits 50 --files
-#     --files costs one extra API request per PR; without it there are no
-#     TOUCHES edges. --no-text skips extraction and writes structure only.
+python scripts/ingest_github.py --repo pallets/click --issues 50 --commits 50 --files --reviews
+#     --files and --reviews each cost one extra request per PR (fetched
+#     concurrently). Without them there are no TOUCHES / REVIEWED edges.
+#     --no-text skips extraction and writes structure only.
+#     50 PRs of pallets/click -> 274 nodes, 572 typed edges in ~60s.
 
 # 3. evaluate (optional)
 python scripts/benchmark.py            # -> results.csv + summary table

backend/scripts/ingest_github.py

@@ -21,6 +21,7 @@
 import os
 import re
 import sys
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 
 import requests
@@ -40,7 +41,10 @@
 logger = logging.getLogger("tracerag.github")
 
 GITHUB_API = "https://api.github.com"
-_PER_PAGE = 100  # GitHub's max page size
+_PER_PAGE = 100          # GitHub's max page size
+_MAX_PR_FILES = 3000     # GitHub's own hard cap on a PR's file list
+_MAX_PR_REVIEWS = 200
+_DETAIL_WORKERS = 8      # concurrent per-PR detail fetches
 
 
 _MD_IMAGE = re.compile(r"!\[[^\]]*\]\([^)]*\)")
@@ -95,6 +99,10 @@ def _paged(url: str, limit: int, token: str | None, params: dict,
         if not batch:
             break
         out.extend(item for item in batch if keep(item))
+        # a short page is the last page — checking this saves one request per
+        # call, which matters when it runs once per PR
+        if len(batch) < _PER_PAGE:
+            break
         page += 1
     return out[:limit]
 
@@ -123,16 +131,40 @@ def fetch_commits(repo: str, limit: int, token: str | None) -> list[dict]:
     return _paged(f"{GITHUB_API}/repos/{repo}/commits", limit, token, {})
 
 
-def attach_pr_files(repo: str, prs: list[dict], token: str | None) -> None:
-    """Fill each PR's `files` in place — one extra request per PR, so opt-in."""
-    for pr in tqdm(prs, desc=f"{repo} files", unit="pr", leave=False):
-        try:
-            pr["files"] = _get(
-                f"{GITHUB_API}/repos/{repo}/pulls/{pr['number']}/files",
-                token, {"per_page": _PER_PAGE},
-            )
-        except Exception as exc:  # noqa: BLE001 — a missing file list isn't fatal
-            logger.debug("[%s] files for #%s failed: %s", repo, pr.get("number"), exc)
+def attach_pr_details(
+    repo: str, prs: list[dict], token: str | None, *,
+    files: bool = False, reviews: bool = False, workers: int = _DETAIL_WORKERS,
+) -> None:
+    """Fill each PR's `files` / `reviews` in place — one request per PR per kind.
+
+    Fetched concurrently: serially this dominated ingest wall time (100 PRs took
+    ~110s of the 143s total). GitHub allows this comfortably inside 5000/hr.
+    """
+    if not (files or reviews):
+        return
+
+    def fetch_one(pr: dict) -> None:
+        number = pr.get("number")
+        if number is None:
+            return
+        base = f"{GITHUB_API}/repos/{repo}/pulls/{number}"
+        if files:
+            # paged: the endpoint caps at 100 per page, so a wide PR would
+            # otherwise be silently truncated at its first hundred files
+            pr["files"] = _paged(f"{base}/files", _MAX_PR_FILES, token, {})
+        if reviews:
+            pr["reviews"] = _paged(f"{base}/reviews", _MAX_PR_REVIEWS, token, {})
+
+    kinds = "+".join(k for k, on in (("files", files), ("reviews", reviews)) if on)
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        futures = {pool.submit(fetch_one, pr): pr for pr in prs}
+        for fut in tqdm(as_completed(futures), total=len(futures),
+                        desc=f"{repo} {kinds}", unit="pr", leave=False):
+            try:
+                fut.result()
+            except Exception as exc:  # noqa: BLE001 — one bad PR isn't fatal
+                logger.debug("[%s] detail fetch for #%s failed: %s",
+                             repo, futures[fut].get("number"), exc)
 
 
 def repo_db_path(repo: str, graphs_dir: Path) -> Path:
@@ -154,6 +186,11 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
     p.add_argument("--files", action="store_true",
                    help="Fetch each PR's changed files for TOUCHES edges. "
                         "Costs one extra API request per PR.")
+    p.add_argument("--reviews", action="store_true",
+                   help="Fetch each PR's reviews for REVIEWED edges. Costs one "
+                        "extra API request per PR. Without this there are no "
+                        "REVIEWED edges — the PR listing only reports pending "
+                        "review requests, not who actually reviewed.")
     p.add_argument("--no-text", action="store_true",
                    help="Skip the prose/curation pass; write structured edges only.")
     p.add_argument("--db", type=Path, default=None,
@@ -182,8 +219,7 @@ def ingest_repo(
     prs = fetch_merged_prs(repo, args.limit, token)
     issues = fetch_issues(repo, args.issues, token)
     commits = fetch_commits(repo, args.commits, token)
-    if args.files:
-        attach_pr_files(repo, prs, token)
+    attach_pr_details(repo, prs, token, files=args.files, reviews=args.reviews)
     logger.info(
         "[%s] fetched %d PRs, %d issues, %d commits -> %s",
         repo, len(prs), len(issues), len(commits), db_path.name,
@@ -225,9 +261,11 @@ def ingest_repo(
             repo, graph.nodes, graph.edges,
             dict(sorted(graph.by_relation.items(), key=lambda kv: -kv[1])),
         )
-        if not args.files:
-            logger.info("[%s] no TOUCHES edges from files — rerun with --files "
-                        "to link PRs to the code they changed.", repo)
+        for flag, enabled, relation in (("--files", args.files, "TOUCHES"),
+                                        ("--reviews", args.reviews, "REVIEWED")):
+            if not enabled:
+                logger.info("[%s] no %s edges — rerun with %s (one extra "
+                            "request per PR).", repo, relation, flag)
     finally:
         db.close()
 

backend/tests/test_db_integration.py

@@ -194,6 +194,32 @@ def test_expand_frontier_carries_relation_and_ts(db):
     assert nbrs[0]["ts"] == NOW - 3 * DAY
 
 
+def test_a_wide_diff_does_not_hide_a_pr_from_traversal(db):
+    """Regression: --files gave real PRs enough TOUCHES edges to trip the hub
+    filter, so 47% of pallets/click's PRs became untraversable."""
+    add(db, "pr", "PR #1", "PR")
+    add(db, "author", "vishal", "Person")
+    db.add_relationship("author", "pr", relation=config.RELATION_AUTHORED)
+    for i in range(30):  # a wide refactor
+        add(db, f"f{i}", f"src/mod{i}.py", "File")
+        db.add_relationship("pr", f"f{i}", relation=config.RELATION_TOUCHES)
+
+    nbrs = db.expand_frontier(["pr"], k=5, max_degree=10).get("pr", [])
+    relations = {n["relation"] for n in nbrs}
+    assert relations == {config.RELATION_AUTHORED}, (
+        "the over-wide TOUCHES fan-out should drop, the author link should not")
+
+
+def test_a_true_hub_is_still_skipped(db):
+    """Every relation fanning out too wide means the node carries no signal."""
+    add(db, "repo", "acme/api", "Repo")
+    for i in range(30):
+        add(db, f"p{i}", f"PR #{i}", "PR")
+        db.add_relationship(f"p{i}", "repo", relation=config.RELATION_TOUCHES)
+        db.add_relationship(f"p{i}", "repo", relation=config.RELATION_PART_OF)
+    assert db.expand_frontier(["repo"], k=5, max_degree=10) == {}
+
+
 def test_subgraph_carries_relation_for_the_canvas(db):
     add(db, "a", "A", "Person")
     add(db, "b", "B", "PR")
@@ -215,7 +241,7 @@ def test_vector_search_carries_ts(db):
 PULL = {
     "number": 412, "title": "Fix token refresh", "merged_at": "2026-07-27T10:00:00Z",
     "user": {"login": "vishal"},
-    "requested_reviewers": [{"login": "arush"}],
+    "reviews": [{"user": {"login": "arush"}}],
     "body": "Fixes #77",
     "files": [{"filename": "auth/session.py"}],
 }

backend/tests/test_github_graph.py

@@ -38,7 +38,7 @@ def build():
     "merged_at": "2026-07-24T09:12:31Z",
     "created_at": "2026-07-23T10:00:00Z",
     "user": {"login": "lena"},
-    "requested_reviewers": [{"login": "marco"}],
+    "reviews": [{"user": {"login": "marco"}}, {"user": {"login": "marco"}}],
     "body": "Routine upgrade. Fixes #2291 and closes #2280.",
     "files": [{"filename": "services/payment/auth.py"}],
 }
@@ -129,3 +129,39 @@ def test_structural_edges_outweigh_proximity():
 
 if __name__ == "__main__":
     pytest.main([__file__, "-q"])
+
+
+def test_requested_reviewers_are_not_counted_as_reviews(build):
+    """`requested_reviewers` is a pending ask, not a completed review — crediting
+    it would claim someone reviewed code they never opened."""
+    db, b = build
+    from tracerag.github_graph import GraphStats
+    b.add_pull_request(
+        {"number": 1, "user": {"login": "lena"},
+         "requested_reviewers": [{"login": "nobody"}]}, GraphStats())
+    assert db.rel(config.RELATION_REVIEWED) == []
+
+
+def test_self_approval_is_not_a_review(build):
+    db, b = build
+    from tracerag.github_graph import GraphStats
+    b.add_pull_request(
+        {"number": 2, "user": {"login": "lena"},
+         "reviews": [{"user": {"login": "lena"}}]}, GraphStats())
+    assert db.rel(config.RELATION_REVIEWED) == []
+
+
+def test_repeat_reviews_collapse_to_one_edge(build):
+    db, b = build
+    from tracerag.github_graph import GraphStats
+    stats = GraphStats()
+    b.add_pull_request(
+        {"number": 3, "user": {"login": "lena"},
+         "reviews": [{"user": {"login": "marco"}}] * 4}, stats)
+    assert len(db.rel(config.RELATION_REVIEWED)) == 1
+    assert stats.by_relation[config.RELATION_REVIEWED] == 1
+
+
+def test_naive_timestamps_are_read_as_utc():
+    """A tz-less string would otherwise take the machine's offset and shift ages."""
+    assert parse_ts("2026-01-01T00:00:00") == parse_ts("2026-01-01T00:00:00Z")

backend/tests/test_ingest_github_wiring.py

@@ -29,7 +29,7 @@
     "number": 412, "title": "Fix token refresh", "merged_at": "2026-07-27T10:00:00Z",
     "html_url": "https://github.com/acme/api/pull/412",
     "user": {"login": "vishal"},
-    "requested_reviewers": [{"login": "arush"}],
+
     "body": "Fixes #77",
 }
 ISSUE = {"number": 77, "title": "Sessions drop", "created_at": "2026-07-20T08:00:00Z",
@@ -38,6 +38,7 @@
           "commit": {"message": "hotfix, closes #77",
                      "author": {"date": "2026-07-27T09:00:00Z"}}}
 FILES = [{"filename": "auth/session.py"}]
+REVIEWS = [{"user": {"login": "arush"}}]
 
 
 # --- fetch filters --------------------------------------------------------
@@ -70,7 +71,7 @@ def boom(*a, **k):
 
     monkeypatch.setattr(gh, "_get", boom)
     prs = [dict(PULL)]
-    gh.attach_pr_files("acme/api", prs, None)  # must not raise
+    gh.attach_pr_details("acme/api", prs, None, files=True)  # must not raise
     assert "files" not in prs[0]
 
 
@@ -81,14 +82,17 @@ def ingested(tmp_path, monkeypatch):
     monkeypatch.setattr(gh, "fetch_merged_prs", lambda *a, **k: [dict(PULL)])
     monkeypatch.setattr(gh, "fetch_issues", lambda *a, **k: [ISSUE])
     monkeypatch.setattr(gh, "fetch_commits", lambda *a, **k: [COMMIT])
-    monkeypatch.setattr(gh, "attach_pr_files",
-                        lambda repo, prs, token: [pr.update(files=FILES) for pr in prs])
+    monkeypatch.setattr(
+        gh, "attach_pr_details",
+        lambda repo, prs, token, **kw: [pr.update(files=FILES, reviews=REVIEWS)
+                                        for pr in prs])
     # stub the embedder — this test is about wiring, not vectors
     monkeypatch.setattr(gh.CurationEngine, "embed", lambda self, text: list(EMBED))
 
     db_path = tmp_path / "acme.lbug"
     args = gh.parse_args(["--repo", "acme/api", "--no-text",
-                          "--issues", "5", "--commits", "5", "--files"])
+                          "--issues", "5", "--commits", "5",
+                          "--files", "--reviews"])
     gh.ingest_repo("acme/api", db_path, args, None, None)
 
     db = TraceDB(db_path, pool_size=1)
@@ -134,3 +138,30 @@ def test_no_relation_is_left_untyped(ingested):
 
 if __name__ == "__main__":
     pytest.main([__file__, "-q"])
+
+
+def test_paging_stops_on_a_short_page(monkeypatch):
+    """A short page is the last one. Without this every per-PR fetch paid for an
+    extra round trip just to see an empty list."""
+    calls = []
+
+    def fake_get(url, token, params=None):
+        calls.append(params["page"])
+        return [{"number": i} for i in range(30)]  # < per_page
+
+    monkeypatch.setattr(gh, "_get", fake_get)
+    assert len(gh.fetch_commits("acme/api", 500, None)) == 30
+    assert calls == [1]
+
+
+def test_a_wide_pr_file_list_is_not_truncated_at_one_page(monkeypatch):
+    """The files endpoint caps at 100 per page; a big refactor must still be
+    fully read rather than silently losing everything past the first hundred."""
+    pages = {1: [{"filename": f"a{i}.py"} for i in range(100)],
+             2: [{"filename": f"b{i}.py"} for i in range(40)]}
+
+    monkeypatch.setattr(gh, "_get",
+                        lambda url, token, params=None: pages.get(params["page"], []))
+    prs = [dict(PULL)]
+    gh.attach_pr_details("acme/api", prs, None, files=True)
+    assert len(prs[0]["files"]) == 140

backend/tracerag/db.py

@@ -383,10 +383,23 @@ def expand_frontier(
             })
         out: dict[str, list[dict[str, Any]]] = {}
         for from_id, nbrs in grouped.items():
-            if len({n["id"] for n in nbrs}) > max_degree:
-                continue  # hub: reached but not traversed through
-            nbrs.sort(key=lambda n: (-n["confidence"], n["id"]))
-            out[from_id] = nbrs[:k]
+            # Hubness is per relation, not per node. A repo TOUCHES everything,
+            # so that relation says nothing specific and is dropped — but the
+            # repo's other edges still work. Judging the node as a whole would
+            # hide any PR with a wide diff, which is the opposite of the point.
+            by_relation: dict[str, list[dict[str, Any]]] = {}
+            for n in nbrs:
+                by_relation.setdefault(n["relation"], []).append(n)
+            kept = [
+                n
+                for group in by_relation.values()
+                if len({x["id"] for x in group}) <= max_degree
+                for n in group
+            ]
+            if not kept:
+                continue  # every relation fans out too wide: a true hub
+            kept.sort(key=lambda n: (-n["confidence"], n["id"]))
+            out[from_id] = kept[:k]
         return out
 
     def documents_for_entities(self, entity_ids: list[str]) -> dict[str, list[dict]]: