diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 000000000..9e868c96e --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,55 @@ +name: Publish to PyPI + +# Publishes graphifyy to PyPI via OpenID Connect (trusted publishing) — no API +# token or password. Fires when a GitHub Release is published (i.e. after +# `gh release create ... --latest`), builds the sdist + wheel, guards that the +# package version matches the release tag, then uploads with an OIDC token. +# +# One-time PyPI setup (Manage project -> Publishing -> Add a GitHub publisher): +# Owner: Graphify-Labs +# Repository: graphify +# Workflow name: publish.yml +# Environment name: pypi +on: + release: + types: [published] + # Manual re-run escape hatch (still requires the release tag to be checked out). + workflow_dispatch: + +permissions: + contents: read + +jobs: + publish: + name: Build and publish graphifyy + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/graphifyy/ + permissions: + id-token: write # REQUIRED: mint the OIDC token PyPI verifies. Nothing else. + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v6 + + - name: Build sdist + wheel + run: uv build --sdist --wheel + + - name: Guard — package version must match the release tag + if: github.event_name == 'release' + run: | + VERSION=$(grep -m1 '^version = ' pyproject.toml | sed -E 's/^version = "(.*)"/\1/') + TAG="${GITHUB_REF_NAME#v}" + echo "pyproject version: $VERSION | release tag: $TAG" + if [ "$VERSION" != "$TAG" ]; then + echo "::error::pyproject version ($VERSION) does not match release tag ($TAG); refusing to publish." + exit 1 + fi + + - name: Twine metadata check + run: uvx twine check dist/* + + - name: Publish to PyPI (trusted publishing) + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/CHANGELOG.md b/CHANGELOG.md index fe0bc82e7..0ce3fdb7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,307 @@ Full release notes with details on each version: [GitHub Releases](https://github.com/safishamsi/graphify/releases) +## 0.9.28 (2026-07-27) + +- Fix: incremental extraction no longer drops cross-file edges whose target file wasn't in the batch (#2211, #2213). Python relative imports and markdown reference links emitted absolute-path-derived target ids without the `target_file` stamp the incremental canonicalization needs, so a re-extracted file's imports/references dangled or vanished; both now stamp the resolved target and canonicalize to the root-relative node. +- Fix: incremental extraction no longer prunes alive files as "deleted" (#2210). The stale-source check compared graph paths to the scan with a raw string test (no Unicode NFC normalization) and pruned any non-match without checking whether the file still exists, so macOS NFD paths and legacy basename spellings lost their nodes. It now compares NFC on both sides and is fail-closed: a source missing from the scan is pruned only when its exclusion is provable, otherwise kept with a warning. +- Fix: `graphify benchmark`, the graph merge-driver, and the call-flow HTML export no longer crash or silently fail on a `--no-cluster` `graph.json` (#2212). Those graphs store edges under `edges` rather than `links`; a shared loader now normalizes both. +- Fix: `claude`/`gemini`/`codebuddy` uninstall no longer delete the user-global skill when called with a `project_dir` (#2215). A passed `project_dir` now scopes the removal; this also fixes a CLI bug where `graphify uninstall --project` deleted the global codebuddy skill. +- Fix: `--update` on macOS no longer re-extracts everything when the corpus path or a filename contains non-ASCII characters (#2221, thanks @SyedFahad7). Manifest keys are NFC-normalized so NFD and NFC path forms match. +- Fix: Swift computed and observed properties (`var body: some View { ... }`, `get`/`set`, `willSet`/`didSet`) now emit graph nodes, so SwiftUI views are no longer erased (#2181, thanks @ozdemirsarman). +- Fix: incremental rebuilds no longer reuse stale community labels, and a graph that outgrows the visualization cap now keeps an aggregated view instead of deleting `graph.html` (#2218, thanks @bobspryn). + +## 0.9.27 (2026-07-26) + +- Fix: `claude`/`gemini`/`codex`/`codebuddy install` no longer overwrite an existing settings/hooks file they cannot parse (#2167). The installers fell back to an empty config on any JSON parse error and then rewrote the whole file, destroying the user's settings (the likely trigger is a UTF-8 BOM, the same class as #2163). They now read `utf-8-sig`, refuse to modify a file that is not a JSON object (naming the path) instead of clobbering it, and back up to `.graphify-bak` before any modifying write. +- Fix: incremental `extract --no-cluster` no longer overwrites the full graph with just the changed files (#2169). The raw path wrote only the current run's extraction over `graph.json`, dropping every node and edge owned by an unchanged file, and the changed file's cross-file edges dangled on absolute-path ids. It now merges the existing graph forward with the same replace/prune semantics as the clustered path and canonicalizes cross-file edge targets; a corrupt existing graph is refused rather than overwritten. +- Fix: Python decorators now create graph edges, so `affected ` finds everything a decorator touches (#2154, thanks @Rishet11). Builtin/stdlib decorators (`@property`, `@staticmethod`, `@dataclass`, `@functools.wraps`, ...) are excluded so they do not fabricate stub nodes or false edges. +- Fix: JavaScript/TypeScript non-relative imports resolve through `jsconfig.json`/`tsconfig.json` `baseUrl` and `paths` (#2153, thanks @Rishet11), so imports like `import x from 'src/utils'` are no longer left dangling. +- Fix: Swift/Foundation/SwiftUI builtins (`Data`, `URL`, `Sendable`, `View`, ...) are filtered from call resolution and god-node ranking (#2147, thanks @MasterFede5), so they no longer fabricate cross-file edges to user symbols or dominate the graph's high-degree nodes. +- Fix: running the test suite no longer touches the developer's real `~/.claude`/`~/.gemini`/`~/.codebuddy`/`~/.copilot` (#2168). An autouse fixture sandboxes HOME for every test. +- Fix: bash calls into functions from a `source`d file now resolve for extensionless shebang scripts and bare `source lib.sh` (no `./` prefix) too (#2171, thanks @Souptik96). The bare-name binding is marked INFERRED since it resolves via `$PATH` at runtime. +- Fix: a `source "${VAR}/lib.sh"` whose variable is a tracked top-level assignment (or the `dirname "${BASH_SOURCE[0]}"` idiom) now resolves against the variable's real directory instead of guessing the script's own, so it no longer binds to a same-named decoy under the script dir (#2172, thanks @Souptik96). +- Fix: SQL `CREATE FUNCTION`/`PROCEDURE` routines with PL/pgSQL-only bodies that tree-sitter cannot parse are now recovered by a raw-text scan, gated on a failed parse so a cleanly-parsing file cannot fabricate routines from commented-out DDL or `EXECUTE` strings (#2180, thanks @Souptik96). +- Fix: the extraction process pool is skipped when only one worker is available, falling back to in-process sequential extraction (#2173, thanks @Souptik96), which also removes the last case where a hung Windows hook rebuild could orphan a worker. +- Fix: the git-hook interpreter pin now handles a Python path containing a space (#2166, thanks @Souptik96); the merge-driver command is quoted and the pin allowlist admits a space while still rejecting shell metacharacters. +- Docs: the Codex `PreToolUse` hook (`graphify hook-check`) is documented as an intentional no-op (#2165, thanks @Souptik96); Codex Desktop rejects `additionalContext` there, so graph guidance comes from `AGENTS.md`. +- Fix: JavaScript/TypeScript regex-rescued imports (Svelte/Astro/Vue) no longer create ghost target nodes with absolute-path ids; the target resolves to the canonical root-relative file node and the dangling absolute-id edges are gone (#2195). +- Fix: cross-file `concept` nodes whose labels normalize identically are now merged, matching the behavior already applied to near-identical labels (#2182). Gated to `concept` nodes with provenance, so code/rationale/document/image and cross-repo guards are preserved. +- Fix: `graphify-out/cache/stat-index.json` is now stored with root-relative keys (re-anchored on load, mirroring `manifest.json`) and pruned of deleted-file entries, so a moved or cloned corpus keeps its cache hits instead of re-extracting everything, and the index stops growing unbounded (#2199). +- Fix: absolute `source_file` paths (for example from a Windows scan) no longer break node identity: `build_from_json` re-keys absolute-derived semantic ids to the canonical root-relative id and the semantic cache stores normalized paths (#2197). +- Fix: `build_from_json` now folds legacy field aliases (`name`->`label`, `path`->`source_file`, edge `type`->`relation`, `confidence_score`->`confidence`), so nodes carrying them stop entering the graph without a label or source_file where they are invisible and unmergeable (#2194). The extraction warning also breaks issues down by cause. +- Feat: C# member calls on a typed receiver now resolve namespace-aware (a class name used in two namespaces resolves via `using`/scope instead of bailing), with `base.`/`this.field` receivers, inherited-member lookup through the `inherits` chain, and shadow-poisoning so a local shadowing a field of a different type no longer produces a wrong edge (#1609, adapted from #1620 by @TheFedaikin). + +## 0.9.26 (2026-07-25) + +- Fix: `graphify query`/`explain` no longer fabricate `indirect_call` edges to class definitions (#2137, thanks @Rishet11). The callable guard admitted classes, so passing a class as a value (`select(Model)`, `db.get(Model, id)`, `except (ErrorA, ErrorB)`, `getattr(obj, "Name", 0)`) produced a false inferred call edge in both the intra-file and cross-file paths. Classes are now tracked separately and excluded from `indirect_call`; direct instantiation still emits its `calls` edge. The suppression is context-blind, so a genuine higher-order class callback that is actually invoked (e.g. `map(Point, coords)`) also loses its edge, which is the intended tradeoff. +- Fix: the post-commit hook's interpreter allowlist now accepts Windows backslash paths (#2126, thanks @Rishet11). The shell `case` glob silently emptied any interpreter path containing a backslash (and the shebang-launcher allowlist admitted no `:` or `\` at all), so the hook failed on Windows uv/venv installs. Both allowlist sites now use a verified character class that admits backslashes while still rejecting shell metacharacters. +- Fix: the hook rebuild timeout is now armed on Windows (#2148, thanks @Rishet11). It relied on `signal.SIGALRM`, which does not exist on Windows, so `GRAPHIFY_REBUILD_TIMEOUT` was a silent no-op and a hung rebuild ran unbounded. A `threading.Timer` fallback now terminates a runaway rebuild where SIGALRM is unavailable; the Unix path is unchanged. +- Fix: bash calls into functions defined in a `source`d file now get `calls` edges (#2141, thanks @HerenderKumar). Call resolution was gated on same-file definitions, so a call to a sourced-library function looked like an external command and produced no edge. Both `source file` and `. file` are handled; resolution is in-corpus and single-match only, so a genuine external command still matches nothing and fabricates no edge. +- Fix: bash `source` edges built from a variable path now resolve (#2079, thanks @HerenderKumar). `source "${BENCH_DIR}/lib/x.sh"` baked the unexpanded `${VAR}` into a dead node id; the leading expansion is now stripped and the literal suffix resolved against the script's directory, emitted as INFERRED only when it resolves to a real file. Calls into a `${VAR}`-sourced library resolve too, not just the source edge. +- Fix: ignore files saved with a UTF-8 BOM are now honored (#2163). `.gitignore`/`.graphifyignore`/`info/exclude` were read as `utf-8`, so a leading BOM stayed on the first line and silently dropped the first pattern (and turned a BOM'd first comment into a bogus pattern). The two ignore read sites now use `utf-8-sig`, matching git, which strips a single leading BOM. + +## 0.9.25 (2026-07-22) + +- License: the project is now licensed under the Apache License, Version 2.0 (previously MIT). Apache 2.0 adds an explicit patent grant and patent-retaliation clause and explicit contribution terms. Contributions made before the relicensing were submitted under MIT and remain available under those terms; the original MIT license text is retained in `LICENSE-MIT` and referenced from `NOTICE`. +- Removed: `.graphifyinclude` handling is gone (#2112). The file has been non-functional since dot directories became indexed by default (#873): the loader and its matchers had no consumers, so `detect` parsed the file on every run and then ignored it, and a `.graphifyinclude` was silently a no-op. The dead loader and matchers are deleted, a leftover `.graphifyinclude` no longer shows up in the `unclassified` list, and `detect` prints a one-time stderr note when one is present at the scan root. To re-include ignored paths, use `!` negation patterns in `.graphifyignore`. + +## 0.9.24 (2026-07-22) + +- Fix: the XAML code-behind `.cs` scan is now bounded and prunes noise dirs, so it can't hang. `_xaml_csharp_class_nodes` used `rglob("*.cs")` over a project root resolved by walking up for a `.csproj`/`.sln`; a standalone `extract_xaml` on a `.xaml` under a large or shared parent (a temp dir, a big monorepo) could resolve the root to a broad ancestor and then recursively scan the whole tree. It now walks with `node_modules`/`.venv`/`.git`/dot-dir pruning and a directory cap, so a real project scans fully while a runaway root degrades to a fast partial scan. + +- Fix: the sensitive-file filter no longer silently drops topic docs and real source (#2106). A prose file whose slug merely ends in a keyword (`privacy-tokens.md`, `token-economics.md`) and real source like `service_account.py` were dropped from the graph with no trace; the filter also missed genuine secrets (`.npmrc`, `.pypirc`, `secring`, `.git-credentials`, case variants). `service_account`/`aws_credentials` moved to the boundary-checked keyword path (real source spared, downloaded key files still excluded), a prose-note carve-out was added (multi-word slugs indexed, bare `secrets.md`/`token.md` still dropped), and the missed secret dotfiles are now caught — net stricter on real secrets while ending the false-positive loss. `graphify extract` and the skill flow now name the skipped-sensitive files instead of only a count, so a wrongly-flagged file is visible. + +- Fix: `calls` edges now resolve through an aliased Python import (#2082, thanks @Yyunozor). `from pkg import mod as alias` (and `import pkg.mod as alias`) recorded the import edge but dropped every downstream `alias.func()` call, so the callee looked like dead code even though the import graph looked complete. The local alias binding is now tracked and the call resolves to the real callee (in-corpus only, so external/stdlib aliases still fabricate nothing). This is the "invisible caller" family that distinguishes graphify from grep/AST name matching. +- Fix: `dedup` (default on) preserves a node's attributes when two exact-ID records from the same source file collapse (#2091, thanks @Synvoya). The collapse discarded one record's fields (e.g. an AST node's `source_location` or a semantic node's `summary`), so the default path silently lost data the docs promised was merged. Non-conflicting attributes are now retained deterministically (independent of chunk order), records from different files or with no source path stay isolated, and a dropped record can never stamp a false `_origin` onto the survivor. +- Fix: the `claude-cli` backend now reads the CLI's structured-output channel instead of trusting free-form prose (#2076, thanks @Yyunozor). Newer Claude Code treats the extraction prompt as an agentic task and reports a summary in the `result` field, which parsed to zero nodes and bisected forever. The backend now pins a JSON schema when the CLI supports it (feature-detected, with the old prompt as the fallback for older CLIs) and parses the `structured_output` object. +- Fix: `graphify explain` on a high-degree node now groups the cut connections by file instead of a bare `... and N more`, so the callers/callees beyond the top 20 are still visible (their file, direction, and count) without a repo-wide grep (#2009, thanks @Yyunozor). +- Fix: `graphify query` no longer prints `calls` edges backwards (#2080, thanks @Yyunozor). The graph on disk is correct, but the CLI loads it undirected and BFS/DFS collect edges in traversal order, so seeding on the callee rendered `callee --calls--> caller`. The renderer now recovers the stored direction from the edge's `_src`/`_tgt` (ignoring stray/dangling values), and both CLI `query` and MCP `query_graph` show the real direction. +- Feat: `get_neighbors` and `get_community` (MCP) now honor a `token_budget` (default 2000) instead of rendering unbounded, so one call on a god node or large community can't flood the client's context (#2069, thanks @ojmucianski). Truncation is announced at the top of the output (matching `query`), with a line count and a narrowing hint. +- Docs: `--code-only` is now surfaced in the `extract` usage text and README (#2071, thanks @HerenderKumar); documented as an `extract` flag rather than a `/graphify` skill flag. +- Docs: README troubleshooting note for an older `graphifyy` in system site-packages shadowing `uv run --with graphifyy`, which silently runs the old version (#1540, thanks @HerenderKumar). + +## 0.9.23 (2026-07-21) + +- Fix: caller / "call sites" listings now report the actual call-site line, not the caller function's definition line. `explain`, `affected`, and the MCP `get_neighbors`/`query` tools printed the caller node's `source_location` (its `def` line) for an incoming call, so a precise-looking citation sent users to the wrong line. The `calls` edge already carries the true call-site line; every caller/relation listing now reads the traversed edge's `source_file`:`source_location`, falling back to the node's own line only when the edge has none. +- Fix: `query` no longer silently drops the answer past its output budget. Rendered nodes were ordered by degree (so a low-degree definition node ranked last and was cut first), the queried symbol was not guaranteed to appear, and the truncation marker sat only at the end so silence read as absence. Nodes are now ranked by hop distance from the query seeds (deterministically), the seed the question named is always rendered first and never truncated, and a prominent notice at the TOP states how many of how many nodes were shown and how to widen the budget. (A branch merge had also silently dropped the seed-first ordering the renderer already supported; it is rewired.) +- Fix: `graphify uninstall` no longer deletes a user-authored `### graphify` section (#2062). The uninstall strip used an unanchored `## graphify` pattern that matched inside a user's H3 heading (and the "already installed" guard was a substring test), so hand-written content was destroyed. The heading is now matched only when a line is exactly the marker (mirroring the install-side #1688 hardening), across all six strip sites (CLAUDE.md, AGENTS.md, GEMINI.md, copilot-instructions.md, CODEBUDDY.md, and the H1 skill registration). +- Fix: `graphify path` (and the MCP `shortest_path` tool) now return a deterministic route and label each hop with the edge's actual stored relation (#2074). The route was computed over a hash-seeded undirected view, so it varied run-to-run among equal-length paths; and the printed relation was read from an arbitrarily-collapsed parallel edge, so it could show `calls` on a pair that only carries `references`. The traversal is now over a sorted graph, and each hop shows the real relation(s), falling back to an honest `related` when none is stored. +- Fix: `cluster-only --no-label` no longer permanently suppresses real community labels (#2073). It wrote `Community N` placeholders (plus a matching signature) into `.graphify_labels.json`, which the reuse path then treated as fresh forever. Placeholder-only runs no longer persist the sidecar, a stored placeholder is treated as absent so already-polluted graphs self-heal, and the watch/update rebuild got the same treatment. +- Fix: `build_from_json`'s ghost-duplicate merge now keys on the full source path, not the bare basename (#2068). Unrelated nodes from different files sharing a common basename (`index.md`, `README.md`) and a generic label were silently merged onto one survivor with their edges rewired, corrupting multi-corpus doc graphs. The legitimate AST/LLM same-file merge is preserved; cross-directory false merges are eliminated. +- Fix: Python import resolution no longer depends on the scan root (#2072). A src-layout project (code under `src/`) lost most of its `imports`/`imports_from` edges when scanned from the repo root, because absolute imports resolved only against the scan root while file-node ids are scan-root-relative, so the dangling edges were silently dropped. Absolute imports now resolve against nested package roots (detected via the `__init__.py` chain), and import edges are repointed to the real file nodes, so the graph is identical whether scanned from the repo root or from `src/`. + +## 0.9.22 (2026-07-20) + +- Fix: a node whose `source_file` is a URL/virtual scheme (`gdoc://`, `s3://`, `http://`, ...) is no longer evicted on the second `graphify update` (follow-up to #2051). The #2051 disk-absence sweep guarded such sources with a literal `"://"` check, but write-side path normalization collapses the double slash (`gdoc://x` becomes `gdoc:/x`), so the guard missed the node on the next run and dropped it into the disk-absence eviction branch. The scheme is now matched tolerantly (and a Windows drive letter like `C:/` is not misread as remote). +- Fix: a real source directory named `env`/`.env`/`*_env` is no longer silently pruned as a false-positive Python virtualenv (#2058). `detect`'s directory-noise heuristic matched those names before `.graphifyignore` negation and with no trace in any output bucket, so codebases using them as source dirs (common in UVM/ASIC verification) lost large subtrees undetectably. The venv heuristic for those names is now gated on an actual marker (`pyvenv.cfg`, an `activate` script, `lib/python*`, or `conda-meta/`); `venv`/`.venv`/`*_venv` stay name-only, and every pruned-as-noise directory is now recorded in a `pruned_noise_dirs` bucket for traceability. +- Fix: Office (`.docx`/`.xlsx`) and Google-Workspace sidecars are now named from the scan-root-relative path, not the absolute path (#2059). The absolute-path hash salted the sidecar name with the checkout location, so committing `graphify-out/` (a supported workflow) produced a new duplicate `.md` per clone/worktree, each ingested as a distinct source document. The relative hash is stable across checkouts while still disambiguating same-stem files; the Google-Workspace sidecar path additionally gains the NFC normalization it was missing. +- Fix: `serve.py`'s "graph.json is corrupted" recovery message is now reachable (#2005, thanks @kimdzhekhon). `json.JSONDecodeError` subclasses `ValueError`, and the broad `except (ValueError, FileNotFoundError)` clause was ordered first, so a truncated graph printed the bare `Expecting value...` instead of the documented rebuild hint. The `JSONDecodeError` clause now comes first. +- Fix: `graphify god-nodes`/`god_nodes` is now a real CLI subcommand, and `graphify extract --output DIR` is honored as an alias of `--out` (#2004). `god_nodes` was an analyzer, an MCP tool, and a documented capability but had no CLI command; `--output` was silently dropped on `extract` even though `graphify tree` documents it. (The `affected`/reverse-dep import-id mismatch from the same report is tracked separately.) +- Fix: a nested class/object/trait now gets its `contains` edge from the enclosing type instead of the file node (#2040). Across ~19 languages the edge was hard-coded to source from the file, so the containment tree was flat (`file -> Inner`) rather than nested (`file -> Outer -> Inner`); it now sources from the enclosing type when present, with top-level types still contained by the file. +- Fix: file nodes that share a basename now get a directory-qualified label so `explain`/discovery can tell them apart (#2032). In directory-per-entrypoint repos (Supabase Edge Functions, Next.js `page.tsx`, Rust `mod.rs`, Python `__init__.py`) dozens of files named e.g. `index.ts` collided under one label, breaking free-text discovery for exactly those files. Colliding file nodes are relabelled to the shortest unique path suffix (`process-order/index.ts`); unique basenames stay bare, and node ids/edges are unchanged. + +## 0.9.21 (2026-07-20) + +- Fix: `graphify extract` (headless, no `--backend`) now auto-detects Ollama from the standard `OLLAMA_HOST` env var, not only graphify's `OLLAMA_BASE_URL` (#1940, thanks @kimdzhekhon). An explicit `OLLAMA_BASE_URL` still wins; `OLLAMA_HOST` is normalized the way the Ollama client does (adds `http://`, defaults the port to 11434 when omitted, appends the `/v1` OpenAI-compat suffix). Wired through both the client base URL and backend auto-detection, so ollama stays opt-in and never shadows a configured paid key. (Supersedes the vendored-bulk #1966.) +- Fix: a flag-less `graphify extract` now honors the persisted `--exclude` patterns instead of silently re-including them (#2027, thanks @oleksii-tumanov). Mirrors the #1971 gitignore-persistence fix: the exclude set is read from `.graphify_build.json` when `--exclude` is absent and applied to the scan, and a flag-less run no longer clobbers it; an explicit `--exclude` still replaces the persisted list. +- Fix: a pathless `--postgres` extract (introspect a live DB with no filesystem corpus) no longer crashes before introspection (#2030, thanks @oleksii-tumanov). The no-path branch left `detection` unbound; it's now initialized, the semantic-cache prune and manifest writes are guarded so a DB-only run can't wipe the file cache or leave a poisoning manifest, and a stale manifest from a prior filesystem run is invalidated. +- Fix: bare import aliases no longer collapse into file-level self-loops (#2037, thanks @Endogen). A single-file import whose bare stem matched the file's own legacy id was remapped onto the importing file, producing a `source == target` self-loop reported as a 1-file import cycle. `build_from_json` now drops any `imports`/`imports_from`/`re_exports` edge whose endpoints are identical; pre-existing self-loops self-heal on the next rebuild. +- Fix: alias re-exports and imports through a barrel resolve to the defining symbol (#1983, thanks @HerenderKumar). `import { X } from './barrel'` where the barrel re-exports `X` from another module now points at `X`'s real defining node, walking the barrel chain (bounded, cycle-safe). When a barrel re-exports the same local name from two different modules the name is ambiguous and left unresolved rather than guessed, so no wrong edge is fabricated. Builds on #1984. +- Fix: a full `graphify update` now evicts semantic nodes whose non-code source file (a `.txt`/`.pdf`/`.png` with no AST extractor) was deleted from disk (#2051). The corpus sweep only checked files it could re-extract, so a deleted doc's or image's LLM-derived nodes survived indefinitely and were served as authoritative. Disk absence is now used as the deletion signal for such sources; remote and virtual sources (anything with a `://` scheme) are left untouched. +- Fix: an incremental rebuild whose change set names a file that exists but has no AST extractor (a doc, paper, image, or an excluded path) no longer treats it as a deletion (#2056). The change-set loop routed any present-but-untracked file to the deletion path, which both evicted its semantic nodes and disabled the shrink guard that would otherwise have caught the loss. Such files are now preserved; a genuine on-disk deletion is still evicted, and the shrink guard now falls through to its per-source accounting instead of being waved off wholesale by the mere presence of a deletion in the change set. +- Fix: code-typed nodes that the semantic pass surfaces from within a document now count as that document's semantic layer (#2014). A doc represented only by code-typed nodes was not recognized as semantically backed, so a rebuild re-scanned it for headings and dropped those nodes. The doc is now correctly treated as semantic-backed and left alone. +- Fix: the `--update` runbook no longer marks a semantic file as done when its extraction produced no output (#2015). Step 9 stamped the entire detected corpus into the manifest, so a doc, paper, or image whose chunk failed or was omitted was recorded as complete and never re-queued on the next update, losing its content permanently. The runbook now builds the manifest with the same stamping the library uses (only files that actually produced nodes, edges, or hyperedges are stamped; dispatched-but-empty files have their stale hash cleared so they are retried), across the Claude, Aider, and Devin skill bodies and the shared update reference. +- Fix: `build_merge` (the `--update` runbook path) now prunes a deleted file's nodes, edges, and hyperedges regardless of whether their stored `source_file` is absolute or relative (#2012). When the caller passed no scan root, a node that had kept an absolute path slipped past the relative prune set and the deleted file's graph survived silently. Matching is now form-insensitive (raw, normalized-relative, then an absolute-identity fallback), a re-extracted file is still never pruned, and `graphify extract` records the scan root marker after every write so a later update relativizes paths correctly even under a custom `--out`. + +## 0.9.20 (2026-07-18) + +- Fix: the graphify-first search nudge now fires on Claude Code's dedicated **Grep tool**, not just Bash (#1986, thanks @mdshzb04). The installed PreToolUse hook only matched `Bash`, so a `Grep` tool call (whose `tool_input` is `{pattern, path, glob, ...}`, not `{command}`) slipped through and never got nudged toward `graphify query`. The matcher is now `Bash|Grep` and the search guard recognizes the Grep shape; it stays nudge-only (never the strict deny), and the uninstall filters + #1840 gating are unchanged. +- Fix: installed hook commands now use forward slashes in the graphify exe path so Git Bash doesn't strip them (#1987, thanks @varuntej07). On Windows the resolved exe path had backslashes, which Git Bash (how Claude Code shells hooks) treats as escapes and drops, breaking the hook with "command not found". `_resolve_graphify_exe` now normalizes `\` to `/` at the single choke point, covering every emitter (Claude/CodeBuddy PreToolUse, Gemini BeforeTool, Codex); quoting and the `--strict` suffix are preserved and POSIX is unaffected. +- Fix: with `--out`, semantic-cache writes now anchor correctly so the cache round-trips (#1990, #1991, thanks @mdshzb04). The final semantic-cache save resolved a relative `source_file` against the output dir and wrote 0 entries, and per-chunk recovery checkpoints landed in the wrong directory (under the corpus instead of `--out`). Cache entries now key on the scan root (portable, matching #1989) while the cache directory sits at the output root, so `check`/`save`/checkpoint/prune all agree; composes with the #1989 salt-keying and #1939 prompt-fingerprint namespacing. +- Fix: alias-based named re-exports no longer emit dangling absolute-path symbol targets (#1983, thanks @oleksii-tumanov). `export { X as Y } from './mod'` produced a `re_exports` edge whose symbol target was an absolute-path-prefixed id with no matching node — the symbol-level residual left by #1967 (imports-only) and #1976 (file-level). The aliased re-export target is now rewritten to the canonical symbol node when unambiguous; external re-exports and owned ids are left untouched, so no real edge is dropped. + +## 0.9.19 (2026-07-18) + +- Feat: opt-in strict PreToolUse hook that actually makes agents use the graph. The installed Claude Code hook has always *nudged* the agent to run `graphify query` before reading raw files, but a nudge is advisory `additionalContext` the model routinely walks past mid-task. `graphify install --project --strict` (or `graphify claude install --strict`) now installs a hook that *blocks* the first raw source read of a session (`permissionDecision: "deny"`) with a redirect to `graphify query`, then downgrades to the soft nudge — so it fires at most once per session and can never strand the agent (the next read proceeds even if no query ran, or if `graphify query` itself failed). Running any `graphify query`/`explain`/`path` refreshes a short-lived "recently oriented" stamp that suppresses the block. Strict mode is Claude Code only (Bash-grep and Glob stay nudge-only; Gemini/Codex/OpenCode can't hard-block and are unchanged); `GRAPHIFY_HOOK_STRICT=1`/`0` toggles it at runtime without a reinstall. Default installs are unchanged (soft nudge). +- Fix: the PreToolUse hook stops crying wolf (#1840), which applies to the default soft nudge too. It no longer fires for reads of files **outside** the indexed project (a common false trigger, e.g. a `~/.claude/.../SKILL.md` read), and when the graph is **stale for the target file** (the file changed after the last build, or `graphify watch` flagged the tree) it softens to a non-mandatory nudge that suggests `graphify update` instead of demanding the query. Gating is ~3 `stat` calls — no corpus walk — so it stays fast on large monorepos, and fails open on any error. +- Fix: a same-basename cross-extension re-export no longer manufactures a phantom self-cycle (#1814, thanks @Greg-Moskalenko for the report and @alphanury for the fix). A typed `.ts` wrapper that re-exports a hand-written `.mjs` runtime (`export { N } from "./foo.mjs"`) had `foo.ts` and `foo.mjs` collapse onto one base file id (the id stem drops the extension), and while `_disambiguate_colliding_node_ids` correctly salts the two file *nodes* apart (`foo_ts_foo` / `foo_mjs_foo`), the re-export *edge* keyed its target salt by the importer's own source file — mis-pointing the `./foo.mjs` target back at `foo.ts`, a `source == target` self-loop reported as a 1-file import cycle in `GRAPH_REPORT.md`. During disambiguation an import/re-export edge now carries the resolved target file as a *transient* salt key, so the salt lands on the real sibling node (generalizing the C/ObjC `.h`-sibling carve-out from #1475 to every language and to `re_exports`) and the phantom cycle disappears. That hint has no downstream reader and holds an absolute path, so it is popped once consumed and never persisted — and the graph serializer drops it as a backstop — keeping graph.json deterministic and byte-identical across checkout locations. Node ids are unchanged (the residual was purely at the edge layer). One caveat: a graph written by a *pre-fix* build still records the stale self-loop, and because `graphify update` only re-extracts changed files, an unchanged wrapper keeps that edge until it is next edited or a `--force` full rebuild runs — though any stale absolute hint a pre-fix graph happened to persist is dropped on the next build regardless. (The extension-aware-id alternative was rejected: it would rewrite every file and symbol id and force a full-rebuild migration in lockstep with the skill/validation id spec, #1033.) +- Fix: `file_hash`'s stat-index memo is now keyed by the path salt, not the absolute path alone (#1989). The digest salts content with the file's path relative to the scan root (for cache portability), but the memo returned whichever digest was computed first for a given absolute path — so the same file hashed under two different roots (which happens within one `--out` run) got an order-dependent result, and the wrong digest was persisted into `stat-index.json` across runs. Each entry now stores one digest per salt; legacy un-salted entries are recomputed rather than trusted. Digest computation is unchanged, so existing cache entries still hit. +- Fix: `--no-gitignore` extraction opt-out for projects that keep useful code under `.gitignore` (#1971, thanks @JensD-git for the report and @Mzt00 for the fix). The flag disables only VCS ignore rules (`.gitignore` + `$GIT_DIR/info/exclude`); `.graphifyignore`, the sensitive-dir/secret screens (#1666/#1943), and noise-dir pruning all still apply, so `.git/`, `node_modules/`, and real secrets stay out of the graph. The setting persists across `update`/`watch`/hook rebuilds and is no longer clobbered back on by a later flag-less `graphify extract`. +- Fix: `.graphifyignore`/`.gitignore` glob matching now keeps `*` within a single path segment (#1975, thanks @oleksii-tumanov). An anchored `*` used to cross `/` (hand-rolled fnmatch), so an `exclude-all + re-include subtree` pattern collapsed to zero files and `/src/*.py` wrongly ignored nested files. Matching now follows git segment semantics (`*` per segment, `**` spans segments, `dir/` matches directories not same-named files), verified against `git check-ignore`; composes with the #1873 anchor-scoping and #1922 diagnostic. +- Fix: DreamMaker parent-relative `#include` paths are no longer mangled (#1978, thanks @Osamaali313). The extractor used `str.lstrip("./")` — a character-set strip that ate every leading `.`/`/`, so `../shared/base.dm` became `shared/base.dm` (and `.hidden/x` lost its dot), breaking include resolution. It now strips only a leading `./` prefix. +- Fix: `graphify prs` decodes `gh`/`git`/`claude` output as UTF-8 instead of the Windows locale codec (#1980, thanks @Luke J). On Windows `text=True` decoded child output as cp1252, mojibaking or crashing on emoji / non-ASCII names in PR titles and diffs; all subprocess reads now pass `encoding="utf-8", errors="replace"` (matching the #1505 precedent), which also fixes the encode side when feeding a non-ASCII prompt to the claude-cli backend. + +## 0.9.18 (2026-07-17) + +- Fix: an incomplete extraction no longer force-writes a partial graph over a complete one (#1951, thanks @TPAteeq). A crashed AST/semantic pass, a some-chunks-failed run, or a walk that couldn't fully enumerate the corpus (permission-denied subtree) produced a smaller graph that the `to_json(force=True)` path wrote anyway, bypassing the #479 shrink guard; the `--no-cluster` raw dump had no guard at all. Both paths now refuse to overwrite a larger existing graph when the run was incomplete (exit 1, nothing written) unless `--allow-partial` is passed, and a present-but-unparseable existing graph fails closed (a corrupt/mid-write file could be hiding a complete graph). `detect()`'s `walk_errors` now count as incomplete too. +- Fix: `graph.json`, `manifest.json`, and the other JSON artifacts are now written atomically (#1952, thanks @TPAteeq). A kill, OOM, or ENOSPC mid-write used to leave a truncated file — and a truncated `graph.json` then wedged every later run via the shrink guard's fail-safe. Writes now go to a temp file in the same directory and `os.replace` into place (writing *through* a symlink so shared-store setups keep working); the writers the original change missed (the `--no-cluster` dump, `merge-graphs`/`merge-chunks`/`merge-semantic`, the analysis/labels sidecars, and the global graph/manifest) are routed through it too. +- Fix: truncated LLM chunks are no longer promoted to the semantic cache as complete (#1950, thanks @TPAteeq). A chunk that hit the output-token limit and couldn't be split further was cached and manifest-stamped as authoritative, so its incomplete node set replayed forever. Such chunks are now marked partial (treated as a cache miss and re-dispatched), including the common case where the truncation parses to *nothing*: the give-up sites record the chunk's own files independently of parsed items, so a sliced document whose later slice truncated empty is no longer stamped complete, and a clean slice can't re-promote a partial entry. +- Fix: untrusted subagent chunk JSON is validated before `merge-chunks` merges it (#1953, thanks @TPAteeq). A malformed chunk used to crash the whole merge (aborting good chunks) or silently pass an adversarial node id (path-escape) into the graph. Each chunk is now validated (reusing the #825 fragment validator, size-capped, id-charset checked) and a bad one is skipped with a warning rather than aborting; non-numeric token counts can no longer TypeError the merge either. +- Fix: code-typed semantic nodes with no evidence in the source are flagged (#1949, thanks @TPAteeq). The LLM can surface a `file_type:"code"` node from a document whose symbol name never actually appears in that source (an inferred or hallucinated symbol); it's now flagged `verification: "unverified"` (a dedicated node field, reported by `graphify diagnose`) rather than presented as a read fact. The check verifies against the node's label and id, only touches nodes the model itself presented as solid, and never drops a node. +- Fix: `watch`/`update` no longer re-scans and duplicates concept/rationale-only semantic docs on every rebuild (#1954, thanks @jw3b-dev). The #1915 semantic-doc gate in `_rebuild_code` recognized a doc as having a semantic (LLM) layer only via a `file_type=="document"` node, but the extraction spec's preferred shape for a doc full of named concepts is to represent it with ONLY `concept`/`rationale` nodes and no separate `document` node — such a doc never entered `semantic_doc_identities`, stayed in the AST quick-scan set, and got duplicate heading nodes minted on top of its real semantic nodes on every `graphify update`/`watch`/hook rebuild (the #1915 symptom returning for this doc shape). The gate now recognizes the full doc-shaped subset of the canonical six-value `file_type` enum (`document`, `concept`, `rationale`, `paper` — the same set `build.py` already treats as doc-representation types), while `code`/`image` nodes and AST-origin-marked nodes are still excluded, so the pre-#1865 legacy-graph safeguard the gate relies on is unchanged. +- Fix: `save_manifest` no longer seeds a stale `semantic_hash` for a file that was dispatched this run but produced no stamped output, masking an LLM-omitted doc on the next run (#1948, thanks @rsolanilla). A file omitted from the semantic result (a `--force` re-run where the model drops its chunk) is dropped from the `files` dict `cli.py`'s `_stamped_manifest_files()` passes to `save_manifest`, per the #933/#1890 never-stamp-a-failed-chunk contract — but the seed loop that carries forward untouched rows for subset saves (#917) then copied that file's row from the on-disk manifest verbatim, including its `semantic_hash` from an earlier successful run. `detect_incremental(kind="semantic")` compared current content against that inherited hash, found a match, and silently reported the file unchanged — defeating the #1890 retry promise the exact way the issue's manual "blank the hash by hand" workaround worked around. `save_manifest` now accepts a `clear_semantic` set; the seed loop forces `semantic_hash` to `""` for any file in it instead of inheriting the stale value. `cli.py` derives the set as `semantic_files` — what was actually sent to the backend this run (narrowed by the incremental gate and `--code-only`, widened by deep mode) — minus `_stamped_manifest_files()`'s result: dispatched-but-not-stamped, regardless of why. Untouched live files that were never dispatched are deliberately not in the set, so a partial incremental run cannot blank the rest of the corpus's stamps. The set is passed at all three `_save_manifest` call sites. `clear_semantic` defaults to `None` (no-op), so every existing caller and the #917/#1908 seed/pruning behavior for untouched and excluded-but-alive rows is unchanged. +- Fix: the semantic cache no longer replays extractions from an older prompt after an upgrade (#1939, thanks @HunterMcGrew and @SinghAman21). Entries were keyed on `sha256(file content + path)` alone, with no component for the extraction prompt that produced them, so a release that changed the prompt left every unchanged file a cache hit: the run exited 0, `cost.json` looked cheap, and the graph silently carried two prompt generations side by side. Semantic entries are now namespaced by a fingerprint of the extraction prompt (`cache/semantic/p{fingerprint}/`, mirroring the AST cache's `v{version}/` layout), keeping both properties #1252 wanted — entries survive releases that don't touch the prompt, and invalidate only when it actually changed. The fingerprint normalizes line endings so a CRLF checkout doesn't look like a prompt change. Both extraction paths pass their prompt: the Python/CLI path (`llm.py`'s `_EXTRACTION_SYSTEM`, all backends) automatically, and the skill path via a new `prompt_file` argument in Step B0/B3 pointing at the `references/extraction-spec.md` the subagents were handed. Pre-existing entries predate fingerprinting and have unknowable vintage: they are still served rather than re-billing a whole corpus, but `check_semantic_cache` now warns with the count, so the "no signal at all" the report describes becomes a visible one; `--force` (or `GRAPHIFY_FORCE=1`) re-extracts them. Old-fingerprint entries are pruned by liveness only, never swept wholesale the way stale AST versions are — two hosts with different prompts can share one `graphify-out/`, and a wholesale sweep would have each run delete the other's entries. (The two monolith skills, aider and devin, inline their prompt instead of shipping a spec sidecar and stay on the unfingerprinted path for now.) +- Fix: the Stage 1 sensitive-directory check no longer silently drops legitimate source under `secrets/` or `credentials/` directories (#1943, thanks @HerenderKumar). A directory named `secrets/`, `.secrets/`, or `credentials/` is as often a real source package (Go `internal/secrets`, a `credentials/` service module) as a credential store, but `_is_sensitive` pruned everything beneath one wholesale, with no trace and no override. The dir list is now split: dedicated credential stores (`.ssh`, `.gnupg`, `.aws`, `.gcloud`) still drop everything unconditionally, while the ambiguous bare-name dirs spare genuine programming-language source — the same carve-out Stage 3 applies to keyword-named files (#1666), extracted into a shared `_is_graphable_source` predicate so the two stages can't drift. Rescued source still falls through the Stage 2/3 filename screens (`secrets/service_account.py` and `credentials/id_rsa` stay dropped), and data/config formats under those dirs (`secrets/db.json`, `.secrets/token.yaml`) remain flagged — those are exactly the formats credentials ship in. +- Fix: PostgreSQL foreign-key `references` edges are no longer dropped when a routine in the same schema is unparseable (#1854, thanks @sekmur). `pg_introspect` builds one synthetic DDL document and parsed it with the function stubs emitted before the FK `ALTER TABLE`s, so a C-language (or otherwise unparseable) routine's stub parsed as a tree-sitter ERROR node that swallowed the trailing FK statements into the error region, losing every FK edge after it. The FK DDL is now emitted before the function stubs, so table-to-table `references` edges are produced first and can't be eaten by a later unparseable routine. +- Fix: `graphify extract --out ` no longer reduces every node's `source_file` to a bare filename, so a `graph.json` stays resolvable against its scan root (#1941, thanks @JensD-git). `--out` passes the output dir as `cache_root` to relocate the cache, but that value also anchored relativization — so every scanned file failed `relative_to(root)`, fell through to the #1899 out-of-root fallback, tripped its `updepth > 3` walk-up guard, and collapsed to a basename; on Windows an `--out` on another drive hit the cross-drive branch and basenamed unconditionally. `extract()` now takes an explicit `root` anchor the CLI pins to the scan root, independent of where the cache lives (completing the #1774 cache/anchor decoupling and matching `build(root=target)`). Cache location, out-of-root portability (#1899), and the no-`--out`/watch paths are unchanged. A graph already written with basenamed paths does not self-heal on an unchanged corpus (the stale nodes are pruned as out-of-scope, leaving them empty); run `graphify extract --force` once, or re-extract after a file change, to rebuild it correctly. + +## 0.9.17 (2026-07-16) + +- Acknowledgement: Amp (ampcode.com) platform support, which shipped earlier in v8, was contributed by @zuwasi in #948 (the `skill-amp.md` skill and the `graphify amp install` wiring). Belated credit for the work. +- Fix: a missing `manifest.json` no longer degrades `graphify extract --code-only` into a full scan that discards the committed semantic layer (#1925). On a fresh clone (or when the manifest is deliberately untracked because its mtimes churn), the incremental gate required both `manifest.json` and `graph.json`; with only the graph present it fell to a full scan, and under `--code-only` that dropped every doc/paper/image node — silently replacing a curated graph with an AST-only skeleton. An existing `graph.json` is now a sufficient incremental baseline: `detect_incremental` already treats an absent manifest as "everything new / nothing deleted", so `build_merge` + `_stale_graph_sources` preserve files that are merely out of this run's scope while still evicting genuinely deleted sources. +- Fix: hyperedge-only documents are now stamped in the manifest instead of being re-extracted on every run (#1920). `_stamped_manifest_files` (#1897) decided a semantic doc "produced output" by inspecting only `nodes` and `edges`, never `hyperedges`, so a chunk whose only output for a doc was a hyperedge (3+ nodes sharing a concept) left that doc unstamped and perpetually re-queued. Stamping now counts hyperedge output too, mirroring the per-`source_file` keying the semantic cache already uses. +- Fix: the PHP extractor now disambiguates same-named classes across namespaces (#1923). A bare class reference (`extends Page`) collapsed onto the only internal class named `Page`, so `App\Models\Page` and an imported `Filament\Pages\Page` fused into one node, manufacturing a false `inherits`/`imports` edge and a bogus cross-community bridge. A new namespace/`use`-aware resolution pass (mirroring the Java resolver, running before the unique-name rewire) re-points supertype/import references to the real definition, or parks provably-external ones on a fully-qualified stub the bare-name rewire cannot collapse. Plain, non-namespaced PHP is unchanged. +- Fix: `detect()` now records files and directories dropped by a `.gitignore`/`.graphifyignore` rule in a new `ignored` diagnostic field (#1922). The nested-ignore scoping bug itself was fixed in 0.9.16 (#1873); this closes the remaining gap where an ignored path left no trace in any diagnostic, so an over-broad rule looked like a clean scan. Entries are per-directory where a subtree is pruned, keeping the list bounded. +- Fix: `_semantic_id_remap` is now idempotent, so incremental rebuilds stop churning (#1917). When a file's canonical stem contained its own legacy stem as a prefix (parent dir name equals the file stem, e.g. `.claude/CLAUDE.md`, `docs/docs.md`), an already-migrated semantic node id re-matched the legacy branch and gained another stem segment on every build (`claude_x` -> `claude_claude_x` -> ...). Because `_origin` is persisted, every `graphify update` re-fed nodes through the remap, so the ids grew unboundedly and the `same_topology`/`same_graph`/`no_change` short-circuits never fired — rewriting `graph.json` and re-running clustering on every zero-delta update. The remap now skips an id that already carries its canonical stem (mirroring the `graph_has_legacy_ids` check), while a genuine one-time legacy migration still applies. (An already-corrupted graph needs one `graphify extract --force` to reset the grown ids.) +- Perf: `graphify query` now scores the graph once per query instead of T+1 times for a T-term query (#1889 / #1918, thanks @Sirhan1). The per-term-guarantee (#1445) previously re-scored the whole graph once per token; `_score_query` now computes the combined ranking and each token's singleton winner in a single traversal, feeding `_pick_seeds` via `best_seed_by_term`. Behavior is preserved (verified byte-identical against the old per-term scoring across a differential fuzz); ~1.3-1.4x faster and independent of query length. + + +- Fix: `--mode deep` is now effective over a warm cache instead of a silent no-op (#1894). The semantic cache is namespaced by mode (`semantic` vs `semantic-deep`) so a shallow-cached file no longer satisfies a deep run; `graphify extract` gains `--force` (and honors `GRAPHIFY_FORCE`) to bypass the incremental gate and the cache read; and a deep incremental run widens its dispatch to the full live doc set so the deep namespace actually gets populated. Cache prune/clear now sweep both namespaces so the deep cache can't accumulate orphans. (The skill-side flow that passes the mode through is a follow-up; the CLI is complete and backward compatible — the new `mode` argument defaults to the existing behavior.) +- Fix: the semantic cache no longer persists dangling edges/hyperedges (#1916). When a node group was skipped on write (out-of-scope per the batch guard, or a ghost `source_file`), edges/hyperedges in the kept groups that referenced those never-written nodes were saved anyway and re-surfaced on every cache replay. Those references are now pruned at write time (gated on the scoping allowlist, so unscoped callers are unchanged), and `build_from_json` validates hyperedge members against the node set so a dangling hyperedge can't reach `graph.json` even from a live extraction. +- Fix: `graphify update`/watch no longer produces a bloated graph by double-representing documents (#1915). `_rebuild_code` AST-quick-scanned Markdown/doc files and then preserved their existing semantic (LLM) nodes on top, so each doc appeared twice (a real corpus came out ~4x). A doc that already has semantic nodes in the graph is no longer AST-quick-scanned (its semantic nodes are the sole representation), while a doc with no semantic layer still gets the structural quick-scan; incremental rebuilds now preserve a doc's semantic nodes instead of evicting them, and previously-bloated graphs self-heal on the next full rebuild. +- Fix: `.cjs` (explicit CommonJS) files are now recognized as code and parsed with the JavaScript grammar (#1912, thanks @Kookwater). The extension was half-registered (present in some internal maps but missing from the classification and dispatch sets), so `.cjs` files were silently dropped. + + +- Fix: files that become excluded (`.graphifyignore`/`.gitignore`/`--exclude`) are now pruned from both the graph and the manifest instead of lingering (#1908 / #1909). Two coupled gaps: `save_manifest` retained any prior row that still existed on disk (disk-existence, not scan-membership), so an excluded-but-present file was reported as `deleted` on every run; and the incremental prune set was derived from the manifest alone, so a newly-excluded file's stale nodes carried forward from the existing `graph.json` were never removed (the state every 0.9.16 graph is in). Now the incremental prune set also derives from the existing graph's own `source_file`s minus the post-exclude corpus (in-root only), `save_manifest` prunes rows outside the full scan corpus, and `detect_incremental` distinguishes truly-deleted files from excluded-but-present ones (which are no longer misreported as deleted). +- Fix: PostgreSQL PL/pgSQL functions are no longer silently dropped (#1910). `CREATE FUNCTION ... LANGUAGE plpgsql AS $$...$$` with `OUT` params, tagged dollar-quotes, or procedural body statements parses as a tree-sitter `ERROR` node, which the SQL extractor skipped entirely. It now recovers the function/procedure name from an ERROR node via the same regex-fallback pattern the extractor already uses elsewhere, so the function node and its `contains` edge are kept (the unparseable body is left opaque) and surrounding statements are unaffected. + + +- Security/privacy follow-up: nodes whose `source_file` was never dispatched are now dropped from the graph, not just skipped from the cache (#1895). The #1757 guard stopped a mis-attributed node from clobbering another file's cache entry, but the node itself still flowed into `graph.json`; it is now filtered out of the merged result (real-file, non-dispatched attributions only), consistent with the cache rejection. +- Fix: `manifest.json` now records every successfully-extracted file, not just the zero-node ones (#1897). The #933 stamping filter compared root-relative node `source_file`s against absolute `detect()` paths, so it dropped every freshly-extracted semantic document from the manifest and broke the incremental-update baseline. Both sides are now resolved before comparison; genuinely omitted/zero-node docs stay unstamped so they retry. +- Fix: `graphify hook install` now registers the `graph.json` union merge driver that the README and CHANGELOG have long documented (#1902). It writes the `merge.graphify` config via `git config` and an idempotent, append-only `graphify-out/graph.json merge=graphify` line in `.gitattributes`; `uninstall` removes them. +- Fix: `hook install`/`status` no longer print a spurious "could not read core.hooksPath" warning on repos whose `.git/config` contains git-legal duplicate keys (VS Code writes these) (#1907). Config is now resolved via `git rev-parse --git-path hooks` instead of a strict `configparser`, which rejected duplicate keys. +- Fix: `graphify export obsidian` prunes notes for nodes that left the graph instead of merging old and new on re-export (#1896). Only notes graphify itself wrote (tracked in its ownership manifest) are removed, with a vault-containment guard, so user-authored notes are never touched. +- Fix: non-English query sentences no longer pick wrong BFS seeds because their filler words were unfiltered (#1900). The query stopword set now covers German and the major Romance languages (curated to avoid clobbering English content words), so `Wie funktioniert die Authentifizierung?` seeds the keyword, not the stopwords. +- Fix: Python calls to an imported module now resolve (already shipped in 0.9.16); `.skill` files (Markdown-with-frontmatter agent files) are now classified as documents instead of being silently dropped as an unsupported extension (#1901). +- Fix: the `--postgres` missing-driver error now points at the correct PyPI package, `graphifyy[postgres]` (was the nonexistent `graphify[postgres]`) (#1906). + +## 0.9.16 (2026-07-14) + +- Fix: semantic extraction now reconciles dispatched files against returned results, so a document the model silently omits is no longer lost without a trace (#1890). A chunk can return a clean, non-empty response that simply leaves out some of the documents it was given; those docs previously produced no node, no warning, and no cache/manifest stamp, so they were re-dispatched and re-omitted on every run. `extract_corpus_parallel` now diffs the dispatched file set against the `source_file`s that came back, records the gap in `uncovered_files`, and prints a loud warning listing the omitted files. (This is the visibility guard; routing documents through the deterministic extractor so they always get at least a file node is tracked separately.) + +- Fix: close two residual paths where an absolute scan path (including the OS username) still leaked into a committed `graph.json`, completing #1789 (#1899). (a) A reference target outside the scan root (an out-of-root `.csproj` ProjectReference, `.sln` project, or bash `source`) kept its absolute `source_file` and an absolute-derived id, because the relativization post-passes silently skipped anything `relative_to(root)` could not handle; such targets now get a portable walk-up relative path and an `ext_`-namespaced id (bare basename when the target is far outside the corpus or on another drive). (b) A symbol whose name normalizes to nothing (a minified `$` function, a JSONC `"//"` comment key) collapsed `_make_id(stem, name)` down to the bare absolute file stem; those no-signal symbols are now skipped at mint time. + +- Fix: uppercase TypeScript extensions (`.TS`/`.TSX`/`.MTS`/`.CTS`) are now parsed with the TypeScript grammar instead of falling through to the JavaScript grammar, which silently dropped interfaces and type aliases (#1881, thanks @xkam7ar). Detection and dispatch already lowercased, but the grammar selection inside `extract_js` compared the suffix case-sensitively. + +- Fix: Kotlin builtin/stdlib types (`String`, `Int`, `List`, ...) are no longer emitted as `references` edges, matching the existing Java/Python/Go builtin filtering (#1876, thanks @kebwlmbhee). They created false coupling and split clusters on real projects. User types that legitimately share a name (`Result`, framework types) are deliberately not filtered, consistent with the other languages. + +- Fix: the stale/missing-skill version warning now prints to stderr, not stdout, so it no longer pollutes the machine-readable output of `graphify query`/`path` (#1805 / #1893, thanks @Mzt00). Every sibling warning in that check already used stderr. + +- Fix: Python qualified calls to an imported module (`module.function()`) now produce a `calls` edge, matching bare-name calls (#1883). The call was captured correctly but fell through a gap: the shared cross-file pass skips every member call (to avoid god-node blowups from bare method names), and the Python member-call resolver only handled capitalized class receivers (`ClassName.method()`), so a lowercase module receiver was dropped. The resolver now also resolves a lowercase receiver that names a module imported into the caller's file, linking to the single callable that module contains (guarded so `self`/`obj`/local instances and ambiguous names never create a false edge). + +- Fix: `--exclude` patterns now survive into `update`/`watch`/git-hook rebuilds instead of applying only to the initial scan (#1886). The patterns were never persisted, so the first rebuild re-ran `detect()` without them and silently re-indexed the excluded paths. `extract` now records them in a `graphify-out/.graphify_build.json` sidecar and `_rebuild_code` re-applies them on every rebuild (they still layer after `.gitignore`/`.graphifyignore`/`.git/info/exclude`, so they keep winning). Graphs built before this release simply have no sidecar and behave as before. + +- Fix: the dedup ID-collision warning now reports in proportion to what is actually lost, and an ID collision resolves to a deterministic survivor (#1851 / #1852, thanks @bchan84x). Two nodes can mint the same `_` ID when a document merely references an entity defined elsewhere; the old code kept whichever arrived first and always printed a scary "two files with the same name" warning that overstated the loss (edges rewire to the survivor, so a reference collapse loses nothing). Now the node whose `source_file` actually defines the ID wins, a harmless reference collapse is silent, a same-file relabel is a quiet note, and only a genuine cross-file collision warns. The survivor is chosen by a total order (definer first, then the shorter/canonical label, then lexically) so it no longer depends on chunk arrival order, including when several same-file nodes co-define one ID. + +- Fix (regression from 0.9.15): a nested `.gitignore`/`.graphifyignore` no longer applies its patterns outside its own directory (#1873 / #1887 / #1885, thanks @Alwyn93). When 0.9.15 started reading nested ignore files, a non-anchored pattern was still matched against the path relative to the scan root, so a nested bare `*` (a common "ignore this scratch dir" idiom, e.g. what the hypothesis library writes into `.hypothesis/`) matched every file and `detect()` returned zero files, silently producing an empty graph. Patterns are now matched relative to the directory whose ignore file defined them and only apply within that subtree; the anchor directory itself is exempt. + +- Fix (regression from 0.9.15): `graphify update` no longer emits 0 nodes and refuses to overwrite an existing `graph.json` when the source tree contains a nested broad `.gitignore` (#1880). This was the update-layer symptom of the scoping bug above: the zeroed re-scan produced an empty rebuild, which the shrink-guard then correctly refused. With subtree scoping fixed the rebuild sees the real files again; the shrink-guard is unchanged. + +- Fix: a full `graphify update` no longer evicts the LLM semantic edges of a re-extracted document (#1865 / #1868, thanks @xor-xe). Node reconciliation was already provenance-aware (semantic nodes lack the AST `_origin` marker and are preserved), but edge reconciliation evicted by `source_file` alone, so a Markdown/doc file that also has an AST extractor had its `semantically_similar_to`/`conceptually_related_to` edges dropped on an AST-only rebuild that cannot regenerate them, leaving orphaned concept nodes. Edges now carry the same `_origin` marker, and re-extraction replaces only a source's AST-tier edges while its semantic edges survive until a semantic re-extraction supersedes them. Deletion still evicts every tier. + +- Fix: `--cargo` no longer drops a workspace-internal dependency edge that uses Cargo's `package = "..."` rename (#1858 / #1861, thanks @thejesh23). The dependency loop looked up crates by the `[dependencies]` table key, but a renamed entry (`db = { path = "../storage", package = "internal-storage" }`) keys on `db` while the target crate is published as `internal-storage`, so `crate_depends_on` silently never appeared. The lookup now honors the `package` override. + +- Fix: `detect_incremental` re-extracts a legacy-manifest file when its mtime moves backwards, not just forwards (#1859 / #1862, thanks @thejesh23). The legacy float-schema branch used a strict `current_mtime > stored`, so a file restored to an older timestamp (a `git checkout` of an older commit, a tarball restore, `rsync --times`) was treated as unchanged and never re-extracted, leaving the graph reflecting newer content than the corpus on disk. It now compares with `!=`, matching the dict-schema branch; with no stored hash to verify, any mtime delta forces a re-extract, and the next save promotes the entry to the hash-verified dict schema. + +- Fix: the dedup summary line reports the fuzzy-merge count even when there were no exact merges (#1857 / #1860, thanks @thejesh23). The fuzzy branch was nested inside `if exact_merges`, so a doc- or semantic-heavy run that merged only via the cross-file fuzzy pass printed a bare `Deduplicated N node(s).` with no breakdown. Both counts are now reported whenever non-zero. + +- Fix: the incremental semantic-cache checkpoint no longer fails on oversized (sliced) documents (#1870). The 0.9.14 batch-scoping fix built its per-chunk allowlist by reading `FileSlice.rel`, an attribute that does not exist (a `FileSlice` carries its parent file in `.path`), so every chunk containing a sliced document leaked the `FileSlice` object into the allowlist, `save_semantic_cache` raised `TypeError`, and the best-effort handler swallowed it: extraction still finished but those chunks were never checkpointed, so a re-run or a run resumed after a crash/rate-limit re-billed them. The allowlist now resolves each unit through `unit_path`, so a slice maps to its parent file and the checkpoint writes as intended. + +## 0.9.15 (2026-07-13) + +- Fix: detection now honors nested `.gitignore`/`.graphifyignore` files below the scan root, not just those at the scan root and above (#1847, thanks @Mohak-Agrawal). git applies a `.gitignore` to everything under its own directory, but graphify only loaded ignore files from the VCS-root-down-to-scan-root chain — so a `vendor/sub/.gitignore` deeper in the tree was never read and its exclusions leaked into the graph. Each directory's own ignore files are now read during the walk and anchored to that directory, preserving last-match-wins precedence (nearer files win, including over `.git/info/exclude`) and the parent-exclusion rule. + +- Fix: `graphify update` now writes human-readable `community_name` labels, not just numeric community ids (#1808 / #1855, thanks @latreon). The incremental rebuild in `_rebuild_code` called `to_json` without `community_labels`, so every `update`/hook rebuild stripped the labels a `cluster-only` pass had written; a follow-up `cluster-only` restored them and the next rebuild stripped them again. The rebuild now forwards the labels it already computes (hub-derived for a code-only pass) and strips `community_name` from the topology fast-path comparison so the new field doesn't force a needless re-cluster. + +- Security: fix a stored XSS and broken neighbor links in the exported `graph.html` (#1838, thanks @edgestack-ai). The report's neighbor "focus" links dropped an unescaped `JSON.stringify(nid)` into a double-quoted inline `onclick`. Because the stringified value carries its own quotes, the attribute was truncated on every node — so the links never worked — and a node `id`/`label` containing a double-quote broke out of the attribute and injected live handlers. AST node ids are `[a-z0-9_]`-safe, but ids/labels from documents or from titles scraped via `graphify add ` are not, so a hostile source could plant an executable handler into a report opened locally. The id is now carried in an HTML-escaped `data-nid` attribute and dispatched through a single delegated listener, closing the injection and repairing the links. + +## 0.9.14 (2026-07-12) + +- Fix: Visual Studio *solution folder* nodes no longer embed the absolute scan path (including the local username) in their `id` and `source_file` (#1789, thanks @fremat79). A solution folder is a virtual grouping, not a file — VS writes its name as both the display name and the "path" — but `extract_sln` resolved it to an absolute filesystem path anyway and keyed the node id off that. The CLI's id-relativization pass only remaps ids of real files in the scan set, so a virtual folder never matched and its absolute id survived into a committed `graph.json` (e.g. `id=/Users//proj/Plugins` instead of `id=plugins`). Solution folders are now detected (name == path) and keyed off the folder name only; real project files still resolve as before. (The earlier fix covered `.csproj`/`.sln` file nodes but missed the virtual folders — this completes it.) + +- Fix: the CLI no longer crashes with exit code 255 when a downstream reader closes the pipe early (#1807 / #1811, thanks @varuntej07). Truncating output with `head`, PowerShell's `Select-Object -First N`, or `sed q` disconnected the reader mid-write, graphify hit an unhandled `BrokenPipeError` (or `OSError(EINVAL)` on Windows) and exited 255 — so CI wrappers and agent harnesses that both trim output and check the exit code read a successful query as a command failure. An early-closing reader is now treated as success: stdout is flushed inside the guard (piped stdout is block-buffered, so a small output would otherwise only flush at interpreter shutdown, where the error escapes as a noisy "Exception ignored" and a nonzero exit), then redirected to devnull so the shutdown flush can't raise again, and the process exits 0. + +- Fix: `extract()` no longer writes its AST cache into the analyzed source tree (#1774 / #1802, thanks @SimiSips). With no explicit `cache_root`, the cache defaulted to the inferred common parent of the input files — the source tree — so analyzing a read-only or foreign corpus silently created `graphify-out/cache/` inside it. The cache is an output, so it now defaults to the current working directory. Crucially, the cache *location* is decoupled from the key/id *anchor*: the inferred common parent still anchors the content-hash keys, node ids, symbol resolution, and the XAML/C# project-scan boundary, so keys stay relative and portable (shared/CI cache reuse keeps working) and out-of-CWD corpora aren't mis-scanned. An explicit `cache_root` (as the CLI and watcher pass) is unchanged. + +- Fix: `graphify query` no longer floods results with homonymous generic symbols (#1766 / #1832, thanks @devcool20). When dozens of nodes share one generic label — framework route handlers all labelled `GET`/`POST`, a repeated `handler` — they used to consume every BFS seed slot, so the traversal explored many near-identical neighborhoods and drowned out the query's actual target. Seed selection now deduplicates by normalized label (`GET`/`Get`/`get` collapse together), keeping at most one representative per label while still guaranteeing a seed for each distinct query term. The per-label cap is scoping-only — the shared node scorer (also used by `shortest_path`/`explain` endpoint resolution) is left untouched, so those are unaffected. + +- Fix: semantic cache writes are now scoped to the files actually dispatched in each extraction batch (#1757 / #1835, thanks @TPAteeq). A model can attribute a node's `source_file` to another corpus file; `save_semantic_cache` would then replace (or, mid-run, pollute) that other file's complete cache entry with a stray fragment — silently, with no shrink guard. Writes now honor an `allowed_source_files` allowlist: the final CLI write is scoped to the uncached file set, and the per-chunk incremental checkpoint (`extract_corpus_parallel`, the default path for `extract`/`update`) is scoped to that chunk's own files, so an out-of-scope attribution is skipped with a warning instead of clobbering a legitimate entry. + +- Fix: `graphify export graphml` no longer crashes on dict- or list-valued attributes (#1831 / #1830, thanks @hofmockel). `nx.write_graphml` only accepts scalar values, so a per-node `metadata` dict or the graph-level `hyperedges` list raised `GraphML does not support type ` and failed the entire export — on a real ~2,300-node graph, every attempt. `to_graphml` now coerces `None -> ""` and JSON-serializes non-scalars across graph/node/edge scopes (GraphML-native int/float/bool/str pass through unchanged), and writes atomically via a temp file so a failed export no longer leaves a 0-byte `.graphml` that downstream tooling mistakes for a completed one. + +- Fix: `.nox/` (nox virtualenvs) is now skipped during detection alongside `.tox/` (#1804, thanks @igorregoir-lgtm). nox is tox's successor and creates a `.nox/` tree of the same shape, but only `.tox` was in the skip set — so a repo with a nox env got its site-packages fully indexed (one real repo came out 91% venv noise: 6,720 of 7,365 nodes from `.nox/`) and semantic extraction burned tokens reading venv docs. + +- Fix: detection now honors `.git/info/exclude`, not just `.gitignore`/`.graphifyignore` (#1810, thanks @cdahl86-cyber). `info/exclude` is where git records local-only excludes and where `git worktree add` writes nested worktree paths, so a repo can exclude a directory without any `.gitignore` entry. graphify walked straight into those worktree copies and the graph exploded (one repo with 5 worktrees went from ~9,400 nodes / 10 MB to ~210,000 nodes / 311 MB, ~77% duplicate worktree nodes, near the 512 MB cap — regenerated on every commit by the auto-rebuild hooks). The exclude file is loaded at lowest precedence (below every per-directory `.gitignore`/`.graphifyignore`), matching git, so a nearer `!` re-include still wins; the linked-worktree/submodule case (where `.git` is a file) is resolved to the shared common git dir. + +- Fix: the git hooks no longer misbehave inside linked worktrees, and `GRAPHIFY_SKIP_HOOK` now suppresses both hooks (#1809, thanks @cdahl86-cyber; worktree guard co-developed with @Claude-Madera's #1806). Two gaps: (1) `post-checkout` never checked `GRAPHIFY_SKIP_HOOK`, so the var stopped commit-triggered rebuilds but not branch-switch ones; it now honors it like `post-commit`. (2) With `core.hooksPath` shared across worktrees, a commit in any linked worktree fired `post-commit`, which wrote a rogue delta-only `graph.json` into that worktree and raced deploy/CI `git clean` against the detached rebuild (`failed to remove graphify-out/: Directory not empty`). Both hooks now short-circuit in a linked worktree (git-dir != git-common-dir), comparing absolute paths so the primary checkout — where `--git-common-dir` is the relative `.git` — is never false-positived and wrongly skipped. + +## 0.9.13 (2026-07-12) + +- Fix: the query log is now opt-in (off by default) (#1797, thanks @adam-pond-agent). `querylog` wrote every `query`/`path`/`explain` question and corpus path (and full responses if `GRAPHIFY_QUERY_LOG_RESPONSES`) to a default-on, unbounded, fail-silent plaintext file at `~/.cache/graphify-queries.log` — outside any repo's .gitignore/retention, and undocumented, which contradicts graphify's on-device / no-telemetry posture. Logging is now OFF unless you opt in with `GRAPHIFY_QUERY_LOG_ENABLE=1` (default path) or `GRAPHIFY_QUERY_LOG=`; `GRAPHIFY_QUERY_LOG_DISABLE=1` still forces it off. All the query-log env vars are now documented in the README. + +- Fix: a markdown file that went through semantic extraction is no longer duplicated into two disconnected nodes on later `graphify update` (#1799, thanks @jerp86). The semantic pass mints `_doc` while the markdown quick-scan mints the bare ``, so the file's edges split across two twins (a docs->code path query would dead-end on the bare half; centrality and communities split too). `build_from_json` now merges the bare quick-scan node into the semantic `_doc` node when both share the same `source_file` and are `file_type: document`, consolidating their edges/hyperedges onto one node. Gated so an unrelated code symbol `foo` and `foo_doc` never merge. + +- Fix: incremental `graphify update` no longer silently evicts nodes for a file that left the scan corpus but still exists on disk (#1795, thanks @CJNA). `_reconcile_existing_graph` read "source absent from the collected corpus" as "deleted", but that's also what an ignore-rule/filter change looks like (e.g. an upgrade that starts honoring `.gitignore`) — in one 27k-node graph the first rebuild after such an upgrade mass-evicted 655 nodes whose files were present the whole time. Eviction now fails closed: a corpus-absent source is only evicted when `Path(identity).exists()` is False (true deletion), otherwise its nodes/edges/hyperedges are preserved and a loud line reports how many were kept and why. True deletions and renames evict as before; a full `extract --force` still purges deliberate exclusions. + +- Fix: `build_merge` no longer silently deletes a re-extracted file's fresh nodes when that file is also passed in `prune_sources` (#1796, thanks @erichkusuki). A file present in `new_chunks` is being replaced, not deleted, so it's now excluded from the prune set — "replace" wins over a contradictory "delete" of the same source. Previously, following the old edit-workflow (pass the changed file in `prune_sources`) deleted the just-built concept whenever an edit kept a node's label. Genuine deletions (a file in `prune_sources` but not `new_chunks`) still prune. + +- Fix: `graphify path` resolves each endpoint to the first candidate whose label contains every query token, instead of blindly taking the top-scored node (#1785, thanks @CJNA). `_score_nodes`' full-query bonus only fires when the query equals/prefixes a label, so a query that is a token *subset* of the intended label (`"Reject-everything judge"` vs `"Degenerate Reject-Everything Judge"`) got no bonus and a node prefix-matching one rare token could outscore it — anchoring the path on an unrelated, often disconnected node and yielding a false "No path found". When the top candidate already full-matches (the common case) the pick is unchanged. Applied to both the `path` CLI and the MCP shortest-path tool; the close-runner-up ambiguity warning now fires only when the score head is what was actually picked. + +- Fix: the report's "Suggested Questions" weakly-connected-node count now matches its "Knowledge Gaps" count (#1768, thanks @balloon72). `suggest_questions()` omitted the `file_type != "rationale"` filter that `report.py`'s Knowledge Gaps section applies, so the same `GRAPH_REPORT.md` showed two different numbers for the same concept (e.g. 757 vs 245), making a healthy graph look like it had a major documentation gap. Both computations now use the same filter. + +- Fix: Bash scripts that run each other by execution now get a cross-file edge (#1756, thanks @balloon72). `extract_bash` only linked `source x.sh` / `. x.sh`; the two most common forms — `bash x.sh` and `./x.sh` — produced no edge, so execution topology was missing. They now emit a `calls` edge (context `script_invocation`) to the invoked script's entry node when the target resolves to a real file on disk (script runners `bash`/`sh`/`zsh`/`ksh`/`dash` and bare `./x.sh`), skipping missing or shadowed targets. + +- Fix: Ruby `.rake` files are now extracted and participate in Ruby cross-file resolution like `.rb` (#1784, thanks @krishnateja7). `.rake` is plain Ruby but the extension was gated out of seven places (classification, extractor dispatch, the language-name/family maps, the `ruby_member_calls` resolver's suffix set, both `.rb`-suffix filters in `ruby_resolution.py`, and the build repo-tag map), so every rake task was skipped and its calls were invisible. All seven now include `.rake`; `Widget.tally` from a `.rake` task resolves to its `.rb` definition. + +- Fix: cross-module references to a function now resolve to its definition instead of dangling on a name-only stub (#1781, thanks @EmilNyg). `_rewire_unique_stub_nodes` gated merge targets through `_is_type_like_definition`, which rejects any label ending in `)` — so function/method defs could never absorb their reference stubs, and "who references this function" returned nothing on the definition node while a sourceless stub held all the edges. Top-level function defs are now eligible rewire targets when the label match is globally unique, gated by a language-family match with the referrers (a Python `get_db` reference can't bind to a unique Go `get_db()`) and excluding stubs used as a supertype (`inherits`/`implements`/`extends` — you don't inherit from a function). Types are unchanged. + +## 0.9.12 (2026-07-10) + +- Fix: live PostgreSQL introspection (`--postgres`) now emits foreign-key `references` edges under a read-only role (#1746, thanks @rithyKabir). The FK query read `information_schema.referential_constraints`, which is privilege-filtered — a role with only SELECT sees zero FK rows while tables/views/routines still appear, so every `references` edge silently vanished. It now reads the world-readable `pg_catalog.pg_constraint` (keyed by oid, which also fixes same-named constraints on sibling tables cross-matching in the old name-based joins), preserving composite-FK column order via `UNNEST ... WITH ORDINALITY`. + +- Fix: `json_config` no longer emits `imports`/`extends` edges to node IDs it never creates (#1764, thanks @oleksii-tumanov). `package.json` dependencies and `tsconfig.json` `extends`/`$ref` targets produced edges whose endpoint node was absent, so `build_from_json` silently dropped them (the "no matching node id" case is filtered out of real errors) — losing dependency/extends structure on two of the most common files in any JS/TS repo. The extractor now creates the referenced target as a `concept` node before adding the edge. + +- Fix: `graphify update` no longer deletes semantic hyperedges on every run (#1755, thanks @oleksii-tumanov). The AST-only rebuild treated every rebuilt corpus file as grounds to evict hyperedges anchored to it, but the AST pass never re-emits hyperedges, so doc-sourced hyperedges (exactly what semantic extraction produces) were permanently lost on the first `update` after a full build — even a no-op run. Hyperedge eviction is now scoped to genuinely deleted (or symlink-outside) sources, mirroring node/edge handling; replacement-by-id and dangling-member cleanup are unchanged. + +- Fix: Java member calls resolve against the receiver's declared type instead of a bare method-name match (#1696/#1697, thanks @oleksii-tumanov). `gw.charge()` where `gw: PaymentGateway` now binds to `PaymentGateway.charge`, not a same-named `AuditLog.charge` in another file. Explicit-type receivers and `this` are exact; current-class fields, method parameters, and explicitly-typed locals resolve via a method-scoped type table; a missing, ambiguous, inherited, or chained receiver is skipped rather than guessed (same god-node guard as the C#/Swift/Ruby resolvers). Fully-qualified and nested-type receivers are deferred (they need package/nesting-aware type identity). + +- Fix: output/cache artifacts no longer land in the scanned corpus or CWD when `--out`/`--graph` point elsewhere (#1747, thanks @bbqboogiedwonsen). `extract --out ` correctly wrote the graph to `` but `detect()`'s word-count/stat-index cache still created a stray `graphify-out/cache/` inside the corpus (it uses the scan root); it now honors the `--out` dir via a threaded `cache_root`. And `cluster-only --graph /graphify-out/graph.json` wrote `GRAPH_REPORT.md`/labels/analysis/re-clustered graph to the CWD instead of beside the input; it now writes beside `--graph` when that graph lives in a `graphify-out/` dir, while still restoring into the CWD for an archived `backup/graph.json` (#934). + +- Fix: `imports`/`references` edges no longer bind across a language boundary (#1749, thanks @philberndt). The spec already forbids cross-language `calls`, but an unresolved Python `import time` could still resolve by bare stem onto a `src/time.ts` file node — welding a polyglot repo's halves together at a phantom edge (in the reporter's repo, 3 such edges were the *only* thing bridging 2409 Python nodes to 1403 TS nodes, inflating `time.ts` betweenness ~90x and making it the #1 "god node"). The build-time cross-language guard now covers `imports`/`imports_from`/`references` in addition to `calls`, dropping an edge only when both endpoints are known code languages of different interop families (so a config/manifest → code reference is untouched). + +- Fix: files whose extractor bailed out for a missing optional dependency no longer vanish without a trace (#1745, thanks @rithyKabir). `.sql` files (and other extra-gated languages) have a dispatch entry, so the #1689 no-extractor warning can't fire, and `extract_sql` returns an error result when `tree-sitter-sql` is absent, so the #1666 zero-node warning skips it too — the graph built "successfully" while an entire SQL corpus contributed nothing. `extract()` now surfaces these grouped by extension, naming the extra that restores the language (e.g. `pip install "graphifyy[sql]"`). + +- Fix: `build_from_json` is deterministic across process runs again (#1753, thanks @erasmust-dotcom). The ghost-node merge iterated `set(G.nodes())`, so which node survived a `(basename, label)` collision depended on CPython's per-process string-hash seed — rebuilding the same extraction JSON in a fresh process could silently pick a different canonical id (breaking the cluster→relabel workflow with a `KeyError` on an id that vanished). The Pass 1/Pass 2 loops now iterate in sorted order. Additionally, two non-AST (semantic) nodes sharing a key but from *different* files are now treated as distinct concepts and both survive (mirroring the AST/AST ambiguity guard #1257) instead of one arbitrarily merging away; a genuine same-file duplicate still collapses. + +- Fix: a Java field/parameter/return-type reference to a class whose simple name is shared by two modules no longer dangles on a sourceless phantom node (#1744, thanks @aviciot). Both same-named classes already survive as distinct path-scoped nodes, but the cross-module `references` edge was left pointing at a bare no-source stub because `_resolve_java_type_references` re-pointed `implements`/`inherits`/`imports` but not `references` — so a query about the referenced class could miss it. The Java resolver now disambiguates `references` by the importing file's `import` statement (falling back to same-package), mirroring the C# resolver, and drops the orphaned phantom. + +## 0.9.11 (2026-07-08) + +- Fix: file enumeration no longer silently drops a directory subtree. `detect()`'s `os.walk` had no `onerror` handler, so an `os.scandir` failure (a permission error, or a directory created/deleted mid-walk by concurrent writes) was swallowed and that whole subtree vanished from the scan with no log, yielding a silently partial `graph.json`. The walk now records every skipped directory (surfaced in the result's `walk_errors`) and warns to stderr, while still enumerating the rest. Relatedly, `to_json`'s anti-shrink guard (#479) now fails safe: a non-empty but unreadable existing `graph.json` refuses the overwrite (pass `force=True` to override) instead of silently clobbering a good graph; an empty file still proceeds. +- Fix: Pascal/Delphi extractors no longer emit duplicate `method`/`contains`/`inherits` edges. A class method declared in the interface section and defined in the implementation section each emitted an edge to the same node, so ~half of a Pascal graph's method edges were doubled (skewing degree/centrality and tripping the new cross-file resolver's god-node guard). Both extractors now dedup edges on (source, target, relation), mirroring the existing node dedup. +- Fix: Pascal/Delphi call resolution is scoped to the caller's class + inherits chain, and calls to methods inherited across file boundaries now resolve (#1739, thanks @richtext). Both extractors previously resolved every call via a single file-wide `{name: node_id}` dict, so two unrelated classes with a same-named method (property accessors, generated COM/TLB wrappers) collapsed onto whichever was inserted last, producing wrong cross-class `calls` edges. Resolution now walks own-class then ancestor chain then file-level free functions, emitting no edge when ambiguous (same god-node guard as the Ruby resolver). A new corpus-wide resolver (`graphify/pascal_resolution.py`) resolves calls from a descendant to a base-class method declared in a different file (the common generated-base/manual-descendant split). Also stops emitting a duplicate cross-file base-class stub carrying the wrong `source_file`. +- Fix: query ranking no longer lets a lone generic term that exact-matches a short leaf label hijack seed selection in multi-term queries (#1602/#1724, thanks @fkhawajagh). `_score_nodes` scales the per-term exact/prefix tiers by squared term coverage; single-term and full-coverage queries are unchanged. +- Fix: Kotlin enum entries are extracted as nodes with `case_of` edges to their enum (#1700, thanks @ivanzhilovich). Closes the Kotlin half of #1700 (the Java half shipped in 0.9.10 via #1719); `enum class ChatType { NORMAL, GROUP, SYSTEM }` now yields NORMAL/GROUP/SYSTEM nodes and "where is ChatType.X used" works for Kotlin. +- Fix: SKILL.md's POSIX interpreter probe no longer silently falls back to a graphify-less system python (#1735, thanks @mohammedMsgm). Step 1 ran `uv tool run graphifyy python -c ...`, but the `graphifyy` package's executable is `graphify`, so uv treated `python` as a missing `graphifyy` command; `2>/dev/null` hid uv's own `--from` hint, leaving `PYTHON` on an interpreter without graphify. The probe now runs `uv tool run --from graphifyy python -c ...`. The PowerShell path was already correct. + +- Refactor: decomposed the two largest modules into focused, single-responsibility modules — verbatim moves only, every original import path preserved via re-exports, no behavior change (#1737, thanks @TPAteeq). `extract.py` 17,054 → 4,740 LOC (the tree-sitter engine, cross-file resolution, shared models, and 23 language extractors moved under `graphify/extractors/`), `__main__.py` 5,368 → 673 (install/uninstall + CLI dispatch split into `graphify/install.py` and `graphify/cli.py`), `export.py` 1,671 → 962 (HTML + graph-DB exporters under `graphify/exporters/`). Full suite unchanged. +- Fix: `merge-graphs` gives each input a distinct repo tag so same-stem nodes from different source graphs don't collapse (#1729). Two graphs under a same-named repo dir (`src/graphify-out` and `frontend/src/graphify-out`, both → `src`) shared the `src::` prefix, so a backend `src/app.js` and a frontend `App.jsx` (both bare `app`) merged into one node with edges from both — false cross-runtime `path` results. Colliding tags are now widened (`frontend_src`) with an index-suffix backstop, and the command prints a note when it disambiguates. +- Fix: `uninstall` removes the graphify hook/section from Claude's local-only files too (#1731, thanks @TPAteeq). It now cleans `.claude/settings.local.json` and both `CLAUDE.local.md` locations in addition to the standard files, via both `graphify uninstall` and `graphify claude uninstall`. +- Feat: `graphify extract --code-only` indexes code (local AST, no API key) and skips the doc/paper/image semantic pass, so a mixed repo no longer hard-fails when no LLM backend is configured (#1734). Reports what it skipped; the no-key error now points users at the flag. + +## 0.9.10 (2026-07-08) + +- Fix: TS/JS member calls on a builtin-typed receiver no longer collapse onto a same-named user symbol (#1726). `_resolve_typescript_member_calls` matched a receiver's type to a definition by casefolded label, so `x: Date; x.getTime()` bound the caller to a user `class DATE`/`const DATE` in another file — inventing hundreds of phantom `references` edges and a false god node. Builtin-global receiver types (`Date`, `Promise`, `Map`, ...) are now skipped, mirroring the cross-file call guard; genuine user types are unaffected. +- Fix: never bind a cross-file `calls` edge to a definition in a different language family (#1718, thanks @edinaldoof). Name-only matching resolved a TSX callback passed by name to a same-named Kotlin method (and a Python call to a Kotlin fun) — phantom edges the spec forbids. Candidates are now filtered by interop family (JVM, native C-family, JS/TS module graph, ...); unknown families stay permissive. +- Fix: an ambiguous legacy-stem alias in `build_merge` no longer silently merges two unrelated files (#1713, thanks @mallyskies). The `#1504` old-stem alias (`ping.h`/`ping.php` → bare `ping`) resolved by hash-order, riding a dangling edge onto an arbitrary same-named file. Aliases are now committed only when exactly one file claims them; a salted `.h`/`.cpp` file node is recognized as its own claimant so a genuine collision stays ambiguous (and dropped) instead of picking a wrong winner. +- Fix: inline base-class stubs are tagged with `origin_file` (#1707, thanks @mallyskies). Five inheritance handlers built cross-file base-class stubs without `origin_file`, so same-named bases across files collapsed onto one shared stub that could then merge with an unrelated real class (218 wrong `inherits` edges observed). They now route through `ensure_named_node`, which sets the tag. +- Fix: Java enum constants are extracted as nodes with `case_of` edges to their enum (#1719, thanks @ivanzhl). Closes the Java half of #1700; `affected ErrorCode` / "where is ErrorCode.X used" now works for Java. +- Fix: `graphify` rebuilds recover from a deleted hook working directory instead of crashing (#1703, thanks @FranciscoJSBarragan). A detached git hook can inherit a CWD that no longer exists; the rebuild now recovers via `GRAPHIFY_REPO_ROOT` or fails cleanly instead of raising `FileNotFoundError`. +- Feat: the semantic cache is checkpointed per chunk so an interrupted extraction resumes instead of restarting (#1715, thanks @A-Levin). Each completed chunk is unioned into the cache immediately (opt out with `GRAPHIFY_NO_INCREMENTAL_CACHE`); the final write still overwrites authoritatively. +- Docs: `SECURITY.md` no longer claims stdio-only now that an opt-in `--transport http` (binds `127.0.0.1` by default) exists (#1714, thanks @Thizeidler); added tests for `GRAPHIFY_MAX_GRAPH_BYTES` parsing and corrected its unit docstring to binary MiB/GiB (#1722, thanks @Cekaru). + +## 0.9.9 (2026-07-07) + +- Fix: `graphify explain` resolves an exactly-typed punctuated label symmetrically against `norm_label` (#1704). The search term tokenized on `\w+` ("blockStream.ts" -> "blockstream ts", space where the '.' was) while a node's stored `norm_label` keeps punctuation ("blockstream.ts"). The verbatim case was already rescued by the tokenized-label tier, but that broke if a node's `label` and `norm_label` diverged; a punctuation-preserving `norm_query` is now matched against `norm_label` across the exact/prefix/substring tiers (and fed to the trigram prefilter), so it is robust by construction. +- Fix: code files with no AST extractor are surfaced instead of silently dropped (#1689, thanks for the precise root-cause). `.r`/`.R` (also `.ejs`, `.ets`) are in `CODE_EXTENSIONS` so they are counted as code, but there is no extractor for them, so they produced zero nodes with no warning. `extract` now prints a grouped warning ("N file(s) are classified as code but graphify has no AST extractor ...: .r (17)"). Adding a real `tree-sitter-r` extractor remains a follow-up. +- Fix: the AST-extraction progress line keeps a consistent denominator to the end (#1693). Intermediate lines counted against `len(uncached_work)` but the final line switched to `total_files` (which includes cached hits and no-extractor files), so on a large corpus the count appeared to jump upward right after 99%. Both the parallel and sequential final lines now use the `uncached_work` denominator. +- Fix: `GRAPH_REPORT.md` no longer emits dangling `[[_COMMUNITY_*]]` Obsidian wikilinks by default (#1712). The `_COMMUNITY_*.md` notes those links target are only created by the opt-in `--obsidian` export, and the report is written at build time before any export, so on a default run every link dangled (spawning phantom nodes in a vault's graph view, literal brackets elsewhere). The Community Hubs section now renders as plain text by default; the wikilink form is behind an `obsidian=True` opt-in. +- Fix: `.m` files are no longer force-parsed by the Objective-C grammar when they are MATLAB (#1702, thanks @catalystdream for the diagnosis). `.m` is shared by Objective-C and MATLAB, but the dispatch routed every `.m` to `extract_objc`, which turned real MATLAB into garbage nodes/edges. `.m` is now content-sniffed like `.h`: a genuine Objective-C `.m` (with `@implementation`/`@interface`/`@import`/`#import`) still routes to `extract_objc`; a MATLAB `.m` gets no extractor and is surfaced by the #1689 warning rather than mis-parsed. `.mm` is unchanged (unambiguously Objective-C++). A real `tree-sitter-matlab` extractor remains a follow-up. +- Fix: the `/graphify` usage comment in the skill files no longer claims a bare `/graphify` produces an Obsidian vault by default (#1681, thanks for the audit). It now reads "full pipeline on current directory (HTML viz; add `--obsidian` for a vault)", matching Step 6. Fixed at the skillgen source so every generated `skill-*.md` variant carries the corrected comment. +- Feat: files graphify sees but cannot classify are surfaced instead of vanishing (#1692). Extensionless, non-shebang project files (Dockerfile, Gemfile, Makefile, Rakefile, LICENSE, ...) and unsupported extensions previously left no trace at all. `detect` now collects them into an `unclassified` list, and `graphify extract` reports "N file(s) not classified (no supported extension or shebang), skipped: ...". Actually extracting Dockerfile/Makefile-style content remains a follow-up. + +## 0.9.8 (2026-07-06) + +- Fix: the Claude Code / Codebuddy `PreToolUse` and Gemini CLI `BeforeTool` graph-nudge hooks now work on Windows (#522). The hooks were inline POSIX bash (`case/esac`, `[ -f ]`, single-quoted `echo`), which Windows cmd.exe/PowerShell cannot parse — so on Windows the hook failed silently, no "run `graphify query` before grepping/reading raw files" context was injected, and users had to invoke `/graphify` by hand. The detection logic (grep-command match, source-file extension match, skip-if-under-output-dir, graph-exists check) moved into a shell-agnostic `graphify hook-guard ` subcommand invoked via the absolute exe path (the same pattern the codex hook already uses), so the hook parses and runs identically on Windows, macOS, and Linux. Behavior on macOS/Linux is unchanged (byte-identical nudge payload); the graph path now also honors `GRAPHIFY_OUT`. The Gemini `BeforeTool` hook got the same treatment (`graphify hook-guard gemini`), which also removes its dependency on a bare `python` being on PATH. Codex stays a no-op there because Codex Desktop rejects `additionalContext`. +- Fix: `--update`-style section writes to `CLAUDE.md`/`AGENTS.md` no longer corrupt or drop content (#1688, thanks @bdfinst). `_replace_or_append_section` located its managed block by substring (`marker in content`) and `next(... if marker in line)`, so a heading that appeared as a substring of another line (or duplicate headings) matched the wrong offset and the rewrite could truncate the file. It now matches the section heading exactly (`line.strip() == marker`), appends when absent, and prefers the last exact match when several exist, so unrelated content is preserved. +- Fix: token estimation no longer crashes on files containing tiktoken special-token text like `<|endoftext|>` (#1685, thanks @Kyzcreig). `_TOKENIZER.encode(content)` raises `ValueError` by default when the text contains a special token, which aborted packing on docs/corpora that merely mention these strings. Both `encode` sites now pass `disallowed_special=()` so such text is tokenized as ordinary bytes. +- Fix: the Ollama backend no longer multiplies a hang by the retry count (#1686, thanks @Kyzcreig). A stalled local model would wedge for `timeout * (max_retries + 1)`, which with the default 6 retries turned one long stall into a very long one. Ollama now defaults to zero client-side retries (a local model that stalls will not un-stall on retry); set `GRAPHIFY_MAX_RETRIES` to opt back in. Other backends are unchanged. Note: the underlying stall is non-deterministic and driven by the model server, so this bounds the wait rather than eliminating the hang. +- Fix: a truncated or slightly malformed community-labeling reply no longer discards the whole batch (#1690, thanks @vdgbcrypto). `_parse_label_response` now salvages the complete `"id": "name"` pairs from a reply that failed a strict `json.loads` (e.g. a reply truncated mid-object), raising only when no pairs can be recovered. The per-batch token budget was also raised (`256 + 48*n`, was `64 + 24*n`) to give models that prepend a short preamble enough headroom to finish the JSON. The exact provider truncation in the report could not be reproduced without a live key; the parser and budget fixes address the mechanism. +- Fix: cluster-only mode now reports the real token cost of community labeling instead of a hardcoded zero (#1694, thanks @sub4biz). The labeling LLM calls were never accounted for, so `GRAPH_REPORT.md`'s "Token cost" line always read `0 input · 0 output` in cluster-only runs. `_call_llm` now accumulates per-response usage into an optional accumulator that is threaded through the labeling path and surfaced in the report. Backends that do not return usage (the Claude Code CLI) still contribute nothing, which is honest rather than estimated. +- Docs/Feat: `deepseek-v4-flash` (and `v4-pro`) have thinking ENABLED by default; graphify no longer implies otherwise and adds an opt-in `GRAPHIFY_DISABLE_THINKING=1` toggle (#1621, thanks @sub4biz for the empirical testing). Disabling thinking removes a rare reasoning-leak failure mode (which the adaptive extraction/labeling retry already recovers from) but, measured on real corpora, trades it for more frequent benign truncation and measurably lower extraction quality and file coverage — so it stays a documented user choice rather than a forced default. The stale "non-thinking" comment on the built-in deepseek config is corrected. The moonshot (kimi) branch is unchanged (it must disable thinking or content comes back empty). +- Fix: source files are no longer silently dropped during discovery by two over-broad filters (#1666, thanks @krishnateja7 for the precise root-cause). (a) A bare `snapshots/` directory was pruned as a Jest/Vitest artifact, which killed legitimate code namespaces like a Rails `app/services/snapshots/`; it is now pruned only when it actually contains `.snap` files or sits directly under a JS test root (`__snapshots__` stays unconditionally pruned). (b) `_is_sensitive` dropped files on a bare name-keyword hit (`device_token.rb`, `passwords_controller.rb`) even when `classify_file` had already resolved them to source code; a genuine programming-language source file is now exempt from the weak keyword heuristic, while real secret stores in data/config formats (`credentials.json`, `secrets.yaml`, `.env`, `.pem`, ...) are still caught. This is the discovery-layer fix; the 0.9.7 no-cache-on-empty change could not surface these because the files never reached extraction. + ## 0.9.7 (2026-07-06) - Fix: Java standard-library types are no longer emitted as `references` noise (#1603, thanks @NydiaChung). A `_JAVA_BUILTIN_TYPES` skip list now suppresses ubiquitous `java.lang`/`java.util`/`java.io`/`java.time`/`java.math`/`java.nio.file` type names (`String`, `List`, `Map`, `Optional`, `Integer`, `Exception`, ...) at the type-ref walker; they never resolve to a project node, so edges to them were pure noise (mirrors `_GO_PREDECLARED_TYPES`/`_PYTHON_ANNOTATION_NOISE`). Nested user-type generic arguments still resolve: `List` drops the `List` edge but keeps `Item`. diff --git a/LICENSE b/LICENSE index b1d9746fb..d64569567 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,202 @@ -MIT License - -Copyright (c) 2026 Safi Shamsi - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 000000000..b1d9746fb --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Safi Shamsi + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/NOTICE b/NOTICE new file mode 100644 index 000000000..791bf88bb --- /dev/null +++ b/NOTICE @@ -0,0 +1,8 @@ +Graphify +Copyright 2026 Safi Shamsi and the Graphify contributors. + +This product is licensed under the Apache License, Version 2.0 (see LICENSE). + +Portions of this software were contributed under the MIT License prior to the +relicensing and remain available under those terms. The original MIT license +text is retained in LICENSE-MIT. diff --git a/README.md b/README.md index c9e4ad701..a4f188f46 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,34 @@

- Graphify + Graphify

- 🇺🇸 English | 🇨🇳 简体中文 | 🇯🇵 日本語 | 🇰🇷 한국어 | 🇩🇪 Deutsch | 🇫🇷 Français | 🇪🇸 Español | 🇮🇳 हिन्दी | 🇧🇷 Português | 🇷🇺 Русский | 🇸🇦 العربية | 🇮🇷 فارسی | 🇮🇹 Italiano | 🇵🇱 Polski | 🇳🇱 Nederlands | 🇹🇷 Türkçe | 🇺🇦 Українська | 🇻🇳 Tiếng Việt | 🇮🇩 Bahasa Indonesia | 🇸🇪 Svenska | 🇬🇷 Ελληνικά | 🇷🇴 Română | 🇨🇿 Čeština | 🇫🇮 Suomi | 🇩🇰 Dansk | 🇳🇴 Norsk | 🇭🇺 Magyar | 🇹🇭 ภาษาไทย | 🇺🇿 Oʻzbekcha | 🇹🇼 繁體中文 | 🇵🇭 Filipino + Graphify-Labs%2Fgraphify | Trendshift

+
+
Read this in other languages + +🇺🇸 English | 🇨🇳 简体中文 | 🇯🇵 日本語 | 🇰🇷 한국어 | 🇩🇪 Deutsch | 🇫🇷 Français | 🇪🇸 Español | 🇮🇳 हिन्दी | 🇧🇷 Português | 🇷🇺 Русский | 🇸🇦 العربية | 🇮🇷 فارسی | 🇮🇹 Italiano | 🇵🇱 Polski | 🇳🇱 Nederlands | 🇹🇷 Türkçe | 🇺🇦 Українська | 🇻🇳 Tiếng Việt | 🇮🇩 Bahasa Indonesia | 🇸🇪 Svenska | 🇬🇷 Ελληνικά | 🇷🇴 Română | 🇨🇿 Čeština | 🇫🇮 Suomi | 🇩🇰 Dansk | 🇳🇴 Norsk | 🇭🇺 Magyar | 🇹🇭 ภาษาไทย | 🇺🇿 Oʻzbekcha | 🇹🇼 繁體中文 | 🇵🇭 Filipino | 🇮🇱 עברית + +
+
+

- YC S26 - Discord - The Memory Layer - CI PyPI Downloads - Sponsor + Discord LinkedIn - X + YC S26

-Type `/graphify` in your AI coding assistant and it maps your entire project — code, docs, PDFs, images, videos — into a knowledge graph you can query instead of grepping through files. +Type `/graphify` in your AI coding assistant and it maps your entire project (code, docs, PDFs, images, videos) into a **knowledge graph** you can **query instead of grepping** through files. + +- **Code maps for free, fully local.** Code is parsed with tree-sitter AST: deterministic, no LLM, nothing leaves your machine. (Docs, PDFs, images and video use your assistant's model, or a configured API key, for a semantic pass.) +- **Every edge is explained.** Each connection is tagged `EXTRACTED` (explicit in the source) or `INFERRED` (resolved by graphify), so you can tell what was read directly from what was inferred. +- **Not a vector index.** No embeddings, no vector store: a real graph you traverse. Ask a question, trace the path between two things, or explain one concept. + +> Want this always-on, updating in the background across your code, docs, and meetings rather than only on demand? That is what we are building at **[graphify.com](https://graphify.com)**. You can join the waitlist there.

graphify's interactive graph.html showing the FastAPI codebase as a force-directed knowledge graph with a legend of detected communities @@ -27,13 +37,20 @@ Type `/graphify` in your AI coding assistant and it maps your entire project — The FastAPI codebase mapped by graphify. Every node is a concept, colors are detected communities, and the whole thing is clickable in graph.html.

-Works in Claude Code, Codex, OpenCode, Kilo Code, Cursor, Gemini CLI, GitHub Copilot CLI, VS Code Copilot Chat, Aider, Amp, OpenClaw, Factory Droid, Trae, Hermes, Kimi Code, Kiro, Pi, Devin CLI, and Google Antigravity. +**Get started** (30 seconds): + +```bash +uv tool install graphifyy # install the CLI (or: pipx install graphifyy) +graphify install # register the skill with your AI assistant +``` + +Then, in your AI assistant: ``` /graphify . ``` -That's it. You get three files: +That's it. You get **three files**: ``` graphify-out/ @@ -42,12 +59,55 @@ graphify-out/ └── graph.json the full graph — query it anytime without re-reading your files ``` -For a readable architecture page with Mermaid call-flow diagrams, run: +**Works in** Claude Code, Cursor, Codex, Gemini CLI, GitHub Copilot, and 15+ more — [pick your platform](#install). -```bash -graphify export callflow-html +--- + +## See it in action + +

+ graphify path query: a terminal asks for the shortest path between FastAPI and ModelField, and the answer lights up hop by hop across the knowledge graph +

+ +Once the graph is built you query it instead of reading files. Real output, graphify run on the FastAPI codebase shown above: + +```text +$ graphify explain "APIRouter" +Node: APIRouter + Source: routing.py L2210 + Community: 2 + Degree: 47 + +Connections (47): + --> RequestValidationError [uses] [INFERRED] + --> Dependant [uses] [INFERRED] + --> .get() [method] [EXTRACTED] + <-- __init__.py [imports] [EXTRACTED] + ... + +$ graphify path "FastAPI" "ModelField" +Shortest path (3 hops): + FastAPI --uses--> DefaultPlaceholder <--references-- get_request_handler() --references--> ModelField ``` +Every edge carries a **confidence tag** (`EXTRACTED` = explicit in the source, `INFERRED` = derived by resolution), so you can tell what was read directly from what was inferred. `graphify query ""` returns a scoped subgraph for a plain-language question, and `graphify path A B` traces how any two things connect. + +--- + +## What it does + +What you get out of the box: + +| Capability | What you get | +|---|---| +| **God nodes** | The most-connected concepts, so you see what everything flows through | +| **Communities** | The graph split into subsystems (Leiden), with LLM-free labels | +| **Cross-file links** | `calls` / `imports` / `inherits` / `mixes_in` resolved across ~40 languages via tree-sitter AST | +| **Query, path, explain** | Ask a question, trace the path between two things, or explain one concept, all against `graph.json` | +| **Rationale + doc refs** | `# NOTE:` / `# WHY:` comments and ADR/RFC citations become first-class nodes linked to the code | +| **Beyond code** | Docs, PDFs, images, and video/audio all map into the same graph | +| **Local-first** | Code is parsed locally with tree-sitter (no LLM, nothing leaves your machine); only the semantic pass over docs/media calls a backend, and only if you configure one | + --- ## Benchmarks @@ -55,8 +115,8 @@ graphify export callflow-html | Benchmark | Metric | graphify | Field | |---|---|---|---| | LOCOMO (n=300) | recall@10 | **0.497** | mem0 0.048, supermemory 0.149 | -| LOCOMO (n=300) | QA accuracy | **45.3%** | supermemory 49.7%, mem0 27.3% | -| LongMemEval-S (n=50) | QA accuracy | **76%** | tied best | +| LOCOMO (n=300) | QA accuracy | 45.3% | supermemory 49.7%, mem0 27.3% | +| LongMemEval-S (n=50) | QA accuracy | **76%** | tied with dense RAG | | Graph build | LLM credits | **0** | per-token for most systems | Every system ran on the same harness with the same model and budgets, scored by a judge blind-validated against a second judge (90.6% agreement, Cohen's kappa 0.81). Full per-system tables, the code-intelligence result, and reproduction commands: **[BENCHMARKS.md](./BENCHMARKS.md)**. @@ -138,6 +198,8 @@ for example `graphify claude install --project` or `graphify codex install --pro > **Git hooks and uv tool / pipx:** `graphify hook install` embeds the current interpreter path directly into the hook scripts at install time, so the post-commit hook fires correctly even in GUI git clients and CI runners where `~/.local/bin` is not on PATH. If you reinstall or upgrade graphify, re-run `graphify hook install` to refresh the embedded path. +> **Strict mode (Claude Code):** `graphify install --project --strict` makes the assistant actually use the graph. The default install *nudges* it to run `graphify query` before reading files; strict mode *blocks* the first raw source read of a session and redirects it to the graph, then reverts to the nudge (so it fires at most once per session and never gets stuck). Toggle at runtime with `GRAPHIFY_HOOK_STRICT=1`/`0`; the default install is unchanged (soft nudge). +
Pick your platform (20+ assistants, click to expand) @@ -167,7 +229,7 @@ for example `graphify claude install --project` or `graphify codex install --pro | Devin CLI | `graphify devin install` | | Google Antigravity | `graphify antigravity install` | -Codex users also need `multi_agent = true` under `[features]` in `~/.codex/config.toml` for parallel extraction. CodeBuddy uses the same Agent tool and PreToolUse hook mechanism as Claude Code. Factory Droid uses the `Task` tool for parallel subagent dispatch. OpenClaw and Aider use sequential extraction (parallel agent support is still early on those platforms). Trae uses the Agent tool for parallel subagent dispatch and does **not** support PreToolUse hooks — AGENTS.md is the always-on mechanism. +Codex users also need `multi_agent = true` under `[features]` in `~/.codex/config.toml` for parallel extraction. CodeBuddy uses the same Agent tool and PreToolUse hook mechanism as Claude Code. Factory Droid uses the `Task` tool for parallel subagent dispatch. OpenClaw and Aider use sequential extraction (parallel agent support is still early on those platforms). Trae uses the Agent tool for parallel subagent dispatch and does **not** support `PreToolUse` hooks, so AGENTS.md is the always-on mechanism. `--platform agents` (alias `--platform skills`) targets the generic cross-framework [Agent-Skills](https://github.com/anthropics/skills) locations: the spec's user-global `~/.agents/skills/` (read by `npx skills` and spec-compliant frameworks) for a global install, and `./.agents/skills/` for a project (`--project`) install. The bare `graphify install` stays single-platform (Claude Code) by design — use the named `agents` platform when you want the skill discoverable by any framework that reads `.agents/skills`. @@ -236,19 +298,24 @@ Run this once in your project after building a graph: | Devin CLI | `graphify devin install` | | Google Antigravity | `graphify antigravity install` | -This writes a small config file that tells your assistant to consult the knowledge graph for codebase questions — preferring scoped queries like `graphify query ""` over reading the full report or grepping raw files. On platforms that support payload-bearing hooks (Claude Code, Gemini CLI), a hook fires automatically before search-style tool calls (and, on Claude Code, before reading source files one by one via the Read/Glob tools) and nudges your assistant toward the graph path. On the others (Codex, OpenCode, Cursor, etc.), the persistent instruction files (`AGENTS.md`, `.cursor/rules/`, etc.) provide the same query-first guidance. `GRAPH_REPORT.md` is still available for broad architecture review. +This writes a small config file that tells your assistant to consult the knowledge graph for codebase questions, preferring scoped queries like `graphify query ""` over reading the full report or grepping raw files. -**CodeBuddy** does the same two things as Claude Code: writes a `CODEBUDDY.md` section telling CodeBuddy to read `graphify-out/GRAPH_REPORT.md` before answering architecture questions, and installs **PreToolUse hooks** (`.codebuddy/settings.json`) that fire before Bash search commands and file reads, nudging toward `graphify query` instead. +- **Hook platforms** (Claude Code, Gemini CLI): a hook fires automatically before search-style tool calls (and, on Claude Code, before reading source files one by one via the Read/Glob tools) and nudges your assistant toward the graph path. +- **Instruction-file platforms** (Codex, OpenCode, Cursor, etc.): persistent instruction files (`AGENTS.md`, `.cursor/rules/`, etc.) provide the same query-first guidance. -**Codex** writes to `AGENTS.md` and also installs a **PreToolUse hook** in `.codex/hooks.json` that fires before every Bash tool call — same always-on mechanism as Claude Code. +`GRAPH_REPORT.md` is still available for broad architecture review. -To remove graphify from all platforms at once: `graphify uninstall` (add `--purge` to also delete `graphify-out/`). Or use the per-platform command (e.g. `graphify claude uninstall`). +**CodeBuddy** does the same two things as Claude Code: writes a `CODEBUDDY.md` section telling CodeBuddy to read `graphify-out/GRAPH_REPORT.md` before answering architecture questions, and installs `PreToolUse` hooks (`.codebuddy/settings.json`) that fire before Bash search commands and file reads, nudging toward `graphify query` instead. ---- +**Codex** writes to `AGENTS.md`, which is what actually carries the always-on graph guidance on this platform. `graphify codex install` also registers a `PreToolUse` hook in `.codex/hooks.json` (`graphify hook-check`), but that entry is deliberately a **no-op**: Codex Desktop rejects `hookSpecificOutput.additionalContext` on `PreToolUse`, so emitting a nudge there would break Bash tool calls. Unlike Claude Code, where the hook (`graphify hook-guard`) does the nudging, on Codex the hook fires and intentionally does nothing, and `AGENTS.md` is the always-on mechanism. + +**Kilo Code** installs the Graphify skill to `~/.config/kilo/skills/graphify/SKILL.md` and a native `/graphify` command to `~/.config/kilo/command/graphify.md`. `graphify kilo install` also writes `AGENTS.md` plus a native `tool.execute.before` plugin (`.kilo/plugins/graphify.js` + `.kilo/kilo.json` or `.kilo/kilo.jsonc` registration) so Kilo gets the same always-on graph reminder behavior through native `.kilo` config. -**Kilo Code** installs the Graphify skill to `~/.config/kilo/skills/graphify/SKILL.md` and a native `/graphify` command to `~/.config/kilo/command/graphify.md`. `graphify kilo install` also writes `AGENTS.md` plus a native **`tool.execute.before` plugin** (`.kilo/plugins/graphify.js` + `.kilo/kilo.json` or `.kilo/kilo.jsonc` registration) so Kilo gets the same always-on graph reminder behavior through native `.kilo` config. +**Cursor** writes `.cursor/rules/graphify.mdc` with `alwaysApply: true`, so Cursor includes it in every conversation automatically, no hook needed. -**Cursor** writes `.cursor/rules/graphify.mdc` with `alwaysApply: true` — Cursor includes it in every conversation automatically, no hook needed. +To remove graphify from all platforms at once: `graphify uninstall` (add `--purge` to also delete `graphify-out/`). Or use the per-platform command (e.g. `graphify claude uninstall`). + +--- ## What's in the report @@ -277,7 +344,7 @@ To remove graphify from all platforms at once: `graphify uninstall` (add `--purg | Video / Audio | `.mp4 .mov .mp3 .wav` and more (requires `uv tool install graphifyy[video]`) | | YouTube / URLs | any video URL (requires `uv tool install graphifyy[video]`) | -Code is extracted locally with no API calls (AST via tree-sitter). Everything else goes through your AI assistant's model API. +Code is extracted **locally with no API calls** (AST via tree-sitter). Everything else goes through your AI assistant's model API. Google Drive for desktop `.gdoc`, `.gsheet`, and `.gslides` files are shortcut pointers, not document content. To include native Google Docs, Sheets, and Slides @@ -333,6 +400,8 @@ Create a `.graphifyignore` in your project root — same syntax as `.gitignore`, **`.gitignore` is respected automatically.** graphify reads the `.gitignore` in each directory. If a `.graphifyignore` is also present, the two are **merged** — `.graphifyignore` patterns are evaluated last, so they win on conflicts (including `!` negations). Adding a `.graphifyignore` only ever excludes more; it never re-includes a file your `.gitignore` already excluded. Subdirectory scoping works the same way as git — an ignore file only affects its own subtree. +Pass `--no-gitignore` to `graphify extract` when git-ignored generated or transpiled code belongs in the graph. This disables `.gitignore` and `.git/info/exclude`; `.graphifyignore` still applies. + ``` # .graphifyignore node_modules/ @@ -450,9 +519,10 @@ These are only needed for **headless / CI extraction** (`graphify extract`). Whe | `GRAPHIFY_GOOGLE_WORKSPACE` | Auto-enable Google Workspace export | optional — set to `1` | | `GRAPHIFY_TRIAGE_BACKEND` | Backend for `graphify prs --triage` | optional — auto-detected from available keys | | `GRAPHIFY_TRIAGE_MODEL` | Model override for triage | optional — e.g. `claude-opus-4-7` | -| `GRAPHIFY_QUERY_LOG` | Override query log path (default: `~/.cache/graphify-queries.log`) | optional — set to empty or `/dev/null` to silence | -| `GRAPHIFY_QUERY_LOG_DISABLE` | Set to `1` to disable query logging entirely | optional | -| `GRAPHIFY_QUERY_LOG_RESPONSES` | Set to `1` to also log full subgraph responses (off by default) | optional | +| `GRAPHIFY_QUERY_LOG_ENABLE` | Set to `1` to turn on the local query log at `~/.cache/graphify-queries.log` (records each query/path/explain question + corpus path). Off by default — nothing is written unless you opt in (#1797) | optional | +| `GRAPHIFY_QUERY_LOG` | Enable the query log and write it to this path instead of the default | optional — off unless this or `_ENABLE` is set | +| `GRAPHIFY_QUERY_LOG_DISABLE` | Set to `1` to force the query log off (wins over the enable vars) | optional | +| `GRAPHIFY_QUERY_LOG_RESPONSES` | When the log is enabled, also record full subgraph responses (off by default) | optional | | `GRAPHIFY_MAX_GRAPH_BYTES` | Override the 512 MiB graph.json size cap — e.g. `700MB`, `2GB`, or plain bytes | optional — useful for very large corpora | | `GRAPHIFY_LLM_TEMPERATURE` | Override LLM temperature for semantic extraction — e.g. `0.7`, or `none` to omit | optional — auto-omitted for o1/o3/o4/gpt-5 reasoning models | @@ -460,11 +530,11 @@ These are only needed for **headless / CI extraction** (`graphify extract`). Whe ## Privacy -- **Code files** — processed locally via tree-sitter. Nothing leaves your machine. A code-only corpus requires no API key — `graphify extract` runs fully offline. +- **Code files** — processed locally via tree-sitter. Nothing leaves your machine. A code-only corpus requires no API key — `graphify extract` runs fully offline. On a mixed repo, add `--code-only` to index just the code and skip the docs/PDFs/images that would otherwise need an LLM. - **Video / audio** — transcribed locally with faster-whisper. Nothing leaves your machine. - **Docs, PDFs, images** — sent to your AI assistant for semantic extraction (via the `/graphify` skill, using whatever model your IDE session runs). Headless `graphify extract` requires `GEMINI_API_KEY` / `GOOGLE_API_KEY` (Gemini), `MOONSHOT_API_KEY` (Kimi), `ANTHROPIC_API_KEY` (Claude), `OPENAI_API_KEY` (OpenAI), `DEEPSEEK_API_KEY` (DeepSeek), a running Ollama instance (`OLLAMA_BASE_URL`), AWS credentials via the standard provider chain (Bedrock - no API key needed, uses IAM), or the `claude` CLI binary (Claude Code - no API key needed, uses your Claude subscription). The `--dedup-llm` flag uses the same key. - **Data residency** — `graphify extract` auto-detects which provider to use based on which API key is set (priority: Gemini → Kimi → Claude → OpenAI → DeepSeek → Azure → Bedrock → Ollama). For code with data-residency requirements, use `--backend ollama` (fully local) or pass an explicit `--backend` flag. Kimi (`MOONSHOT_API_KEY`) routes to Moonshot AI servers in China. -- No telemetry, no usage tracking, no analytics. +- **No telemetry**, no usage tracking, no analytics. - **Query logging** — every `graphify query`, `graphify path`, `graphify explain`, and MCP `query_graph` call is logged to `~/.cache/graphify-queries.log` in JSON Lines format (timestamp, question, corpus, nodes returned, duration). Full subgraph responses are **not** stored by default. Set `GRAPHIFY_QUERY_LOG_DISABLE=1` to opt out, or `GRAPHIFY_QUERY_LOG=/dev/null` to silence without disabling the code path. --- @@ -480,6 +550,17 @@ The CLI is installed but its bin directory isn't on your shell's `PATH`. Pick th **`uvx graphify …` or `uv tool run graphify …` fails to resolve `graphify`** The PyPI package is `graphifyy`; `graphify` is only the command it provides. `uv tool run` treats the first word as a *package name*, so it looks for a package called `graphify` and reports `No solution found … no versions of graphify`. Name the package explicitly: `uvx --from graphifyy graphify install` (same as `uv tool run --from graphifyy graphify install`). Or `uv tool install graphifyy` once and then call `graphify` directly. +**`uv run --with graphifyy python -m graphify` silently runs an older install** +`uv run` uses your *system* Python, so if an older `graphifyy` also lives there (e.g. a past `pip install graphifyy`), Python can find that copy first on `sys.path` and `--with graphifyy` won't override it. It runs with no error, but you get the *old* version's behavior — e.g. env overrides like `OPENAI_BASE_URL` are silently ignored, so requests hit the default endpoint and fail with a 401 that looks like a bad key. The fingerprint is a `warning: skill is from graphify , package is ` line — that means a different install was loaded, not just a stale skill. Check which copy actually loaded: +```bash +python -c "import graphify; print(graphify.__file__)" +``` +Then run the installed command directly (it uses the uv-managed copy), or drop the stale system copy: +```bash +uvx --from graphifyy graphify extract . --backend openai # names the package explicitly +pip uninstall graphifyy # or remove the old system install +``` + **`python -m graphify` works but `graphify` command doesn't** Your shell's `PATH` doesn't include the bin directory the command was installed to. Prefer `uv tool install` / `pipx install` over plain `pip`, then run `uv tool update-shell` / `pipx ensurepath` and open a new terminal (see the install notes above). @@ -489,6 +570,9 @@ PowerShell treats a leading `/` as a path separator. Use `graphify .` (no slash) **Graph has fewer nodes after `--update` or rebuild** If a refactor deleted files, the old nodes linger. Pass `--force` (or set `GRAPHIFY_FORCE=1`) to overwrite even when the rebuild has fewer nodes. +**`extract` exits with "extraction was incomplete ... refusing to overwrite"** +When an extraction pass crashes or a walk can't fully read the corpus, the run would be smaller than a complete one, so `graphify extract` refuses to overwrite a larger existing graph with the partial result (protecting your `graph.json`). Fix the underlying failure and re-run, or pass `--allow-partial` to overwrite anyway. + **Graph has duplicate nodes for the same entity (ghost duplicates)** Ghost duplicates (same symbol appearing twice — once from AST extraction with a source location, once from semantic extraction without) are now automatically merged at build time. If you see this in a graph built before v0.8.33, run a full re-extract to clean up: ```bash @@ -548,6 +632,7 @@ graphify-out/ /graphify # run on current directory /graphify ./raw # run on a specific folder /graphify ./raw --mode deep # more aggressive relationship extraction +graphify extract ./raw --code-only # index code only — local AST, no API key (skips docs/PDFs/images); an `extract` flag, not a skill flag /graphify ./raw --update # re-extract only changed files /graphify ./raw --directed # preserve edge direction /graphify ./raw --cluster-only # rerun clustering on existing graph @@ -647,6 +732,7 @@ graphify extract ./docs --token-budget 30000 # smaller semantic chunks for loc graphify extract ./docs --max-concurrency 2 # fewer parallel LLM calls (useful for local inference) graphify extract ./docs --api-timeout 900 # longer HTTP timeout for slow local models (default 600s) graphify extract ./docs --google-workspace # export .gdoc/.gsheet/.gslides via gws before extraction +graphify extract ./src --no-gitignore # include git-ignored source; still honor .graphifyignore graphify extract ./docs --mode deep # richer semantic extraction via extended system prompt graphify extract ./docs --no-cluster # raw extraction only, skip clustering graphify extract ./docs --timing # print per-stage wall-clock timings to stderr (also works on cluster-only) @@ -703,16 +789,17 @@ graphify label ./my-project --backend=openai --model gpt-4o # force a specific - [How it works](docs/how-it-works.md) — the extraction pipeline, community detection, confidence scoring, benchmarks - [ARCHITECTURE.md](ARCHITECTURE.md) — module breakdown, how to add a language - [Optional integrations](docs/docker-mcp-sqlite.md) — Docker MCP Toolkit + SQLite +- [The Memory Layer](https://safishamsi.gumroad.com/l/qetvlo) — the book on the ideas behind graphify, the architecture end to end --- -## Built on graphify — Penpax +## graphify Enterprise -[**Penpax**](https://graphifylabs.ai) is the always-on layer built on top of graphify — it applies the same graph approach to your entire working life: meetings, browser history, emails, files, and code, updating continuously in the background. +[**graphify Enterprise**](https://graphify.com) is the always-on layer built on top of graphify — it applies the same graph approach to your entire working context: meetings, files, docs, and code, updating continuously in the background. -Built for people whose work lives across hundreds of conversations and documents they can never fully reconstruct. No cloud, fully on-device. +Built for people and teams whose work lives across hundreds of conversations and documents they can never fully reconstruct. -**Free trial launching soon.** [Join the waitlist →](https://graphifylabs.ai) +**[Join the waitlist at graphify.com](https://graphify.com).** Free trial launching soon. --- @@ -769,8 +856,12 @@ See [ARCHITECTURE.md](ARCHITECTURE.md) for module responsibilities and how to ad --- +## Community and links +

- - Star History Chart - + Website + Discord + X + Sponsor + The Memory Layer

diff --git a/SECURITY.md b/SECURITY.md index 297b7d8d3..795454b21 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -42,7 +42,7 @@ graphify is a **local development tool**. It runs as a Claude Code skill and opt ### What graphify does NOT do -- Does not run a network listener (MCP server communicates over stdio only) +- Does not run a network listener by default (stdio transport); `--transport http` is opt-in, documented in the README, and binds to `127.0.0.1` unless `--host 0.0.0.0` is passed - Does not execute code from source files (tree-sitter parses ASTs - no eval/exec) - Does not use `shell=True` in any subprocess call - Does not store credentials or API keys diff --git a/docs/demo-path.svg b/docs/demo-path.svg new file mode 100644 index 000000000..4bf10a6e8 --- /dev/null +++ b/docs/demo-path.svg @@ -0,0 +1 @@ +graphify$graphifypath"FastAPI""ModelField"Shortest path (3 hops):FastAPI --uses--> DefaultPlaceholder <--references-- get_request_handler() --references--> ModelField3 hops. Zero files opened.usesreferencesreferencesFastAPIDefaultPlaceholderget_request_handler()ModelField \ No newline at end of file diff --git a/docs/translations/README.fa-IR.md b/docs/translations/README.fa-IR.md index 87ae87b6c..2851c0803 100644 --- a/docs/translations/README.fa-IR.md +++ b/docs/translations/README.fa-IR.md @@ -1,5 +1,5 @@

- Graphify + Graphify

@@ -511,11 +511,11 @@ graphify --version ## ساخته‌شده روی graphify — Penpax -[**Penpax**](https://graphifylabs.ai) لایه همیشه‌روشن ساخته‌شده روی graphify است — همان رویکرد گراف را بر کل زندگی کاری شما اعمال می‌کند: جلسات، تاریخچه مرورگر، ایمیل‌ها، فایل‌ها و کد، به‌صورت مداوم در پس‌زمینه به‌روزرسانی می‌شود. +[**Penpax**](https://graphify.com) لایه همیشه‌روشن ساخته‌شده روی graphify است — همان رویکرد گراف را بر کل زندگی کاری شما اعمال می‌کند: جلسات، تاریخچه مرورگر، ایمیل‌ها، فایل‌ها و کد، به‌صورت مداوم در پس‌زمینه به‌روزرسانی می‌شود. ساخته‌شده برای کسانی که کارشان در صدها مکالمه و سند پراکنده است. بدون ابر، کاملاً روی دستگاه. -**آزمایش رایگان به‌زودی راه‌اندازی می‌شود.** [به لیست انتظار بپیوندید ←](https://graphifylabs.ai) +**آزمایش رایگان به‌زودی راه‌اندازی می‌شود.** [به لیست انتظار بپیوندید ←](https://graphify.com) --- diff --git a/docs/translations/README.fil-PH.md b/docs/translations/README.fil-PH.md index 6a8ef4cda..e318eb68e 100644 --- a/docs/translations/README.fil-PH.md +++ b/docs/translations/README.fil-PH.md @@ -72,6 +72,6 @@ Ang mga code file ay prinoseso nang lokal sa pamamagitan ng tree-sitter AST. Ang ## Binuo sa ibabaw ng graphify — Penpax -Ang [**Penpax**](https://graphifylabs.ai) ay ang enterprise layer sa ibabaw ng graphify. **Malapit nang magkaroon ng libreng trial.** [Sumali sa waitlist →](https://graphifylabs.ai) +Ang [**Penpax**](https://graphify.com) ay ang enterprise layer sa ibabaw ng graphify. **Malapit nang magkaroon ng libreng trial.** [Sumali sa waitlist →](https://graphify.com) [![Star History Chart](https://api.star-history.com/svg?repos=safishamsi/graphify&type=Date)](https://star-history.com/#safishamsi/graphify&Date) diff --git a/docs/translations/README.he-IL.md b/docs/translations/README.he-IL.md new file mode 100644 index 000000000..65e9ae75d --- /dev/null +++ b/docs/translations/README.he-IL.md @@ -0,0 +1,849 @@ +

+ Graphify +

+ +

+ 🇺🇸 English | 🇨🇳 简体中文 | 🇯🇵 日本語 | 🇰🇷 한국어 | 🇩🇪 Deutsch | 🇫🇷 Français | 🇪🇸 Español | 🇮🇳 हिन्दी | 🇧🇷 Português | 🇷🇺 Русский | 🇸🇦 العربية | 🇮🇷 فارسی | 🇮🇹 Italiano | 🇵🇱 Polski | 🇳🇱 Nederlands | 🇹🇷 Türkçe | 🇺🇦 Українська | 🇻🇳 Tiếng Việt | 🇮🇩 Bahasa Indonesia | 🇸🇪 Svenska | 🇬🇷 Ελληνικά | 🇷🇴 Română | 🇨🇿 Čeština | 🇫🇮 Suomi | 🇩🇰 Dansk | 🇳🇴 Norsk | 🇭🇺 Magyar | 🇹🇭 ภาษาไทย | 🇺🇿 Oʻzbekcha | 🇹🇼 繁體中文 | 🇵🇭 Filipino | 🇮🇱 עברית +

+ +

+ YC S26 + Discord + The Memory Layer + CI + PyPI + Downloads + Sponsor + LinkedIn + X +

+ +

+ + Star History Chart + +

+ +
+ +הקלידו `‎/graphify` בעוזר ה-AI לכתיבת קוד שלכם, והוא ימפה את הפרויקט כולו — קוד, מסמכים, קובצי PDF, תמונות, סרטונים — לגרף ידע שאפשר לשאול עליו שאלות, במקום לחפש בקבצים עם grep. + +עובד ב-Claude Code, Codex, OpenCode, Kilo Code, Cursor, Gemini CLI, GitHub Copilot CLI, VS Code Copilot Chat, Aider, Amp, OpenClaw, Factory Droid, Trae, Hermes, Kimi Code, Kiro, Pi, Devin CLI ו-Google Antigravity. + +
+ +``` +/graphify . +``` + +
+ +זה הכול. מקבלים שלושה קבצים: + +
+ +``` +graphify-out/ +├── graph.html נפתח בכל דפדפן — לחיצה על צמתים, סינון, חיפוש +├── GRAPH_REPORT.md עיקרי הדברים: מושגי מפתח, קשרים מפתיעים, שאלות מוצעות +└── graph.json הגרף המלא — אפשר לשאול עליו בכל רגע בלי לקרוא שוב את הקבצים +``` + +
+ +לדף ארכיטקטורה קריא עם דיאגרמות זרימת-קריאות ב-Mermaid, הריצו: + +
+ +```bash +graphify export callflow-html +``` + +--- + +
+ +## דרישות מקדימות + +| דרישה | מינימום | בדיקה | התקנה | +|---|---|---|---| +| Python | 3.10+ | `python --version` | [python.org](https://www.python.org/downloads/) | +| uv *(מומלץ)* | כל גרסה | `uv --version` | `curl -LsSf https://astral.sh/uv/install.sh \| sh` | +| pipx *(חלופה)* | כל גרסה | `pipx --version` | `pip install pipx` | + +**התקנה מהירה ב-macOS (עם Homebrew):** + +
+ +```bash +brew install python@3.12 uv +``` + +
+ +**התקנה מהירה ב-Windows:** + +
+ +```powershell +winget install astral-sh.uv +``` + +
+ +**Ubuntu/Debian:** + +
+ +```bash +sudo apt install python3.12 python3-pip pipx +# או התקנת uv: +curl -LsSf https://astral.sh/uv/install.sh | sh +``` + +--- + +
+ +## התקנה + +> **החבילה הרשמית:** חבילת ה-PyPI היא `graphifyy` (עם y כפולה). חבילות `graphify*` אחרות ב-PyPI אינן קשורות לפרויקט. פקודת ה-CLI היא עדיין `graphify`. + +**שלב 1 — התקנת החבילה:** + +
+ +```bash +# מומלץ (סביבה מבודדת; אם הפקודה 'graphify' לא נמצאת אחר כך, הריצו: uv tool update-shell): +uv tool install graphifyy + +# חלופות: +pipx install graphifyy +pip install graphifyy # עשוי לדרוש הגדרת PATH — ראו הערה בהמשך +``` + +
+ +**שלב 2 — רישום המיומנות (skill) אצל עוזר ה-AI שלכם:** + +
+ +```bash +graphify install +``` + +
+ +זהו. פתחו את עוזר ה-AI והקלידו `‎/graphify .` + +כדי להתקין את המיומנות בתוך המאגר הנוכחי במקום בפרופיל המשתמש, הוסיפו `--project`: + +
+ +```bash +graphify install --project +graphify install --project --platform codex +``` + +
+ +התקנה ברמת הפרויקט כותבת לתוך התיקייה הנוכחית, למשל `.claude/skills/graphify/SKILL.md` או `.agents/skills/graphify/SKILL.md` (בתוספת תיקיית `references/` שהמיומנות טוענת לפי הצורך), ומדפיסה רמז `git add` לקבצים שאפשר לבצע להם commit. פקודות פר-פלטפורמה שתומכות בהתקנה ברמת הפרויקט מקבלות את אותו דגל, למשל `graphify claude install --project` או `graphify codex install --project`. + +> **הערת PowerShell:** השתמשו ב-`graphify .` ולא ב-`‎/graphify .` — הלוכסן המוביל הוא מפריד נתיבים ב-PowerShell. + +> **‏`graphify: command not found`?** ‏`uv tool install` / `pipx install` מציבים את פקודת `graphify` בתיקיית הכלים שלהם (`~/.local/bin`). אם המעטפת (shell) לא מוצאת אותה מיד אחרי ההתקנה — נפוץ בהתקנת macOS + zsh טרייה — התיקייה הזו עדיין לא ב-`PATH`: הריצו `uv tool update-shell` (או `pipx ensurepath`) ופתחו טרמינל חדש. עם `pip` רגיל, הוסיפו את `~/.local/bin` (בלינוקס) או `~/Library/Python/3.x/bin` (במק) ל-PATH, או הריצו `python -m graphify`. + +> **מריצים עם `uvx` / `uv tool run` בלי להתקין?** ציינו את שם החבילה, לא את שם הפקודה: `uvx --from graphifyy graphify install`. ‏`uvx graphify …` רגיל נכשל (`No solution found … no versions of graphify`) כי `uv tool run` קורא את המילה הראשונה כשם *חבילה*, והחבילה היא `graphifyy` — פקודת `graphify` נמצאת בתוכה. + +> **הימנעו מ-`pip install` במק/Windows** אם אפשר. המיומנות מאתרת את Python בזמן ריצה מתוך `graphify-out/.graphify_python`; אם הוא מצביע על סביבה שונה מזו שבה `pip` התקין את החבילה, תקבלו `ModuleNotFoundError: No module named 'graphify'`. ‏`uv tool install` ו-`pipx install` מבודדים את החבילה בסביבה משלהם ונמנעים מהבעיה לחלוטין. + +> **הוקים של Git עם uv tool / pipx:** ‏`graphify hook install` מטמיע את נתיב המפרש הנוכחי ישירות בסקריפטי ההוק בזמן ההתקנה, כך שהוק ה-post-commit יופעל כראוי גם בלקוחות Git גרפיים וב-CI שבהם `~/.local/bin` אינו ב-PATH. אם התקנתם מחדש או שדרגתם את graphify, הריצו שוב `graphify hook install` כדי לרענן את הנתיב המוטמע. + +### בחרו את הפלטפורמה שלכם + +| פלטפורמה | פקודת התקנה | +|----------|----------------| +| Claude Code (לינוקס/מק) | `graphify install` | +| Claude Code (Windows) | `graphify install` (זיהוי אוטומטי) או `graphify install --platform windows` | +| CodeBuddy | `graphify install --platform codebuddy` | +| Codex | `graphify install --platform codex` | +| OpenCode | `graphify install --platform opencode` | +| Kilo Code | `graphify install --platform kilo` | +| GitHub Copilot CLI | `graphify install --platform copilot` | +| VS Code Copilot Chat | `graphify vscode install` | +| Aider | `graphify install --platform aider` | +| OpenClaw | `graphify install --platform claw` | +| Factory Droid | `graphify install --platform droid` | +| Trae | `graphify install --platform trae` | +| Trae CN | `graphify install --platform trae-cn` | +| Gemini CLI | `graphify install --platform gemini` | +| Hermes | `graphify install --platform hermes` | +| Kimi Code | `graphify install --platform kimi` | +| Amp | `graphify amp install` | +| Agent Skills (חוצה-פלטפורמות) | `graphify install --platform agents` (כינוי: `--platform skills`) | +| Kiro IDE/CLI | `graphify kiro install` | +| Pi coding agent | `graphify install --platform pi` | +| Cursor | `graphify cursor install` | +| Devin CLI | `graphify devin install` | +| Google Antigravity | `graphify antigravity install` | + +משתמשי Codex צריכים גם `multi_agent = true` תחת `[features]` בקובץ `~/.codex/config.toml` לצורך חילוץ מקבילי. CodeBuddy משתמש באותו מנגנון Agent tool והוק PreToolUse כמו Claude Code. ‏Factory Droid משתמש בכלי `Task` לשיגור תת-סוכנים במקביל. OpenClaw ו-Aider משתמשים בחילוץ טורי (תמיכה בסוכנים מקביליים עדיין מוקדמת בפלטפורמות אלו). Trae משתמש ב-Agent tool לשיגור תת-סוכנים במקביל ו**אינו** תומך בהוקים מסוג PreToolUse — ‏AGENTS.md הוא המנגנון הקבוע שם. + +`--platform agents` (כינוי: `--platform skills`) מכוון למיקומים הגנריים חוצי-הפלטפורמות של [Agent-Skills](https://github.com/anthropics/skills): ‏`~/.agents/skills/` הגלובלי של המשתמש (נקרא על ידי `npx skills` ומסגרות תואמות-מפרט) בהתקנה גלובלית, ו-`./.agents/skills/` בהתקנת פרויקט (`--project`). ‏`graphify install` החשוף נשאר חד-פלטפורמי (Claude Code) בכוונה — השתמשו בפלטפורמת `agents` כשתרצו שהמיומנות תהיה זמינה לכל מסגרת שקוראת `.agents/skills`. + +> Codex משתמש ב-`‎$graphify` במקום `‎/graphify`. + +### תוספים אופציונליים + +התקינו רק מה שצריך: + +| תוסף | מה הוא מוסיף | התקנה | +|---|---|---| +| `pdf` | חילוץ PDF | `uv tool install "graphifyy[pdf]"` | +| `office` | תמיכה ב-`.docx` ו-`.xlsx` | `uv tool install "graphifyy[office]"` | +| `google` | רינדור Google Sheets | `uv tool install "graphifyy[google]"` | +| `video` | תמלול וידאו/אודיו (faster-whisper + yt-dlp) | `uv tool install "graphifyy[video]"` | +| `mcp` | שרת MCP stdio | `uv tool install "graphifyy[mcp]"` | +| `neo4j` | דחיפה ל-Neo4j | `uv tool install "graphifyy[neo4j]"` | +| `falkordb` | דחיפה ל-FalkorDB | `uv tool install "graphifyy[falkordb]"` | +| `svg` | ייצוא גרף ל-SVG | `uv tool install "graphifyy[svg]"` | +| `leiden` | זיהוי קהילות Leiden ‏(Python < 3.13 בלבד) | `uv tool install "graphifyy[leiden]"` | +| `ollama` | הרצה מקומית עם Ollama | `uv tool install "graphifyy[ollama]"` | +| `openai` | OpenAI / ממשקי API תואמי-OpenAI | `uv tool install "graphifyy[openai]"` | +| `gemini` | Google Gemini API | `uv tool install "graphifyy[gemini]"` | +| `anthropic` | Anthropic Claude API ‏(`--backend claude`, משתמש ב-`ANTHROPIC_API_KEY`) | `uv tool install "graphifyy[anthropic]"` | +| `bedrock` | AWS Bedrock (משתמש ב-IAM, ללא מפתח API) | `uv tool install "graphifyy[bedrock]"` | +| `azure` | Azure OpenAI Service ‏(`--backend azure`, משתמש ב-`AZURE_OPENAI_API_KEY` + ‏`AZURE_OPENAI_ENDPOINT`) | `uv tool install "graphifyy[openai]"` | +| `sql` | חילוץ סכמות SQL | `uv tool install "graphifyy[sql]"` | +| `postgres` | אינטרוספקציה של PostgreSQL חי (`--postgres DSN`) | `uv tool install "graphifyy[postgres]"` | +| `dm` | חילוץ AST של BYOND DreamMaker ‏`.dm`/`.dme` (עשוי לדרוש קומפיילר C + ‏`python3-dev` אם אין wheel מתאים לפלטפורמה) | `uv tool install "graphifyy[dm]"` | +| `terraform` | חילוץ AST של Terraform / HCL ‏`.tf`/`.tfvars`/`.hcl` | `uv tool install "graphifyy[terraform]"` | +| `chinese` | פילוח שאילתות בסינית (jieba) | `uv tool install "graphifyy[chinese]"` | +| `all` | כל מה שלמעלה | `uv tool install "graphifyy[all]"` | + +--- + +## גרמו לעוזר שלכם להשתמש בגרף תמיד + +הריצו פעם אחת בפרויקט אחרי בניית גרף: + +| פלטפורמה | פקודה | +|----------|---------| +| Claude Code | `graphify claude install` | +| CodeBuddy | `graphify codebuddy install` | +| Codex | `graphify codex install` | +| OpenCode | `graphify opencode install` | +| Kilo Code | `graphify kilo install` | +| GitHub Copilot CLI | `graphify copilot install` | +| VS Code Copilot Chat | `graphify vscode install` | +| Aider | `graphify aider install` | +| OpenClaw | `graphify claw install` | +| Factory Droid | `graphify droid install` | +| Trae | `graphify trae install` | +| Trae CN | `graphify trae-cn install` | +| Cursor | `graphify cursor install` | +| Gemini CLI | `graphify gemini install` | +| Hermes | `graphify hermes install` | +| Kimi Code | `graphify install --platform kimi` | +| Amp | `graphify amp install` | +| Agent Skills (חוצה-פלטפורמות) | `graphify agents install` (כינוי: `graphify skills install`) | +| Kiro IDE/CLI | `graphify kiro install` | +| Pi coding agent | `graphify pi install` | +| Devin CLI | `graphify devin install` | +| Google Antigravity | `graphify antigravity install` | + +הפקודה כותבת קובץ תצורה קטן שמנחה את העוזר שלכם להתייעץ עם גרף הידע בשאלות על בסיס הקוד — ולהעדיף שאילתות ממוקדות כמו `graphify query "<שאלה>"` על פני קריאת הדוח המלא או grep על קבצים גולמיים. בפלטפורמות שתומכות בהוקים נושאי-מטען (Claude Code, ‏Gemini CLI), הוק מופעל אוטומטית לפני קריאות כלי בסגנון חיפוש (וב-Claude Code גם לפני קריאת קובצי מקור אחד-אחד דרך הכלים Read/Glob) ומכוון את העוזר לנתיב הגרף. באחרות (Codex, ‏OpenCode, ‏Cursor וכו'), קובצי ההנחיות הקבועים (`AGENTS.md`, ‏`.cursor/rules/` וכו') מספקים את אותה הנחיית "קודם הגרף". ‏`GRAPH_REPORT.md` עדיין זמין לסקירת ארכיטקטורה רחבה. + +**CodeBuddy** עושה את אותם שני דברים כמו Claude Code: כותב קטע `CODEBUDDY.md` שמנחה את CodeBuddy לקרוא את `graphify-out/GRAPH_REPORT.md` לפני מענה על שאלות ארכיטקטורה, ומתקין **הוקים מסוג PreToolUse** ‏(`.codebuddy/settings.json`) שמופעלים לפני פקודות חיפוש ב-Bash וקריאת קבצים, ומכוונים ל-`graphify query` במקום. + +**Codex** כותב ל-`AGENTS.md` וגם מתקין **הוק PreToolUse** ב-`.codex/hooks.json` שמופעל לפני כל קריאת כלי Bash — אותו מנגנון קבוע כמו ב-Claude Code. + +להסרת graphify מכל הפלטפורמות בבת אחת: `graphify uninstall` (הוסיפו `--purge` כדי למחוק גם את `graphify-out/`). או השתמשו בפקודה הפר-פלטפורמית (למשל `graphify claude uninstall`). + +--- + +**Kilo Code** מתקין את מיומנות Graphify ל-`~/.config/kilo/skills/graphify/SKILL.md` ופקודת `‎/graphify` נטיבית ל-`~/.config/kilo/command/graphify.md`. ‏`graphify kilo install` כותב גם `AGENTS.md` וגם **תוסף `tool.execute.before` נטיבי** (`.kilo/plugins/graphify.js` + רישום ב-`.kilo/kilo.json` או `.kilo/kilo.jsonc`) כך ש-Kilo מקבל את אותה התנהגות תזכורת-גרף קבועה דרך תצורת `.kilo` נטיבית. + +**Cursor** כותב `.cursor/rules/graphify.mdc` עם `alwaysApply: true` — ‏Cursor מכליל אותו בכל שיחה אוטומטית, ללא צורך בהוק. + +## מה יש בדוח + +- **צומתי מפתח (God nodes)** — המושגים המקושרים ביותר בפרויקט. הכול עובר דרכם. +- **קשרים מפתיעים** — קישורים בין דברים שחיים בקבצים או מודולים שונים. מדורגים לפי מידת ההפתעה. +- **ה"למה"** — הערות בקוד (`# NOTE:`‏, `# WHY:`‏, `# HACK:`), ‏docstrings ורציונל עיצובי מהמסמכים מחולצים כצמתים נפרדים המקושרים לקוד שהם מסבירים. +- **שאלות מוצעות** — 4–5 שאלות שהגרף נמצא בעמדה ייחודית לענות עליהן. +- **תגי ביטחון** — כל קשר מוסק מסומן `EXTRACTED`‏, `INFERRED` או `AMBIGUOUS`. תמיד יודעים מה נמצא ומה נוחש. + +--- + +## אילו קבצים הוא מטפל + +| סוג | סיומות | +|------|-----------| +| קוד (36 דקדוקי tree-sitter) | `.py .ts .js .jsx .tsx .mjs .go .rs .java .c .cpp .h .hpp .cu .cuh .metal .rb .cs .kt .scala .php .swift .lua .luau .zig .ps1 .psm1 .ex .exs .m .mm .jl .vue .svelte .astro .groovy .gradle .dart .v .sv .svh .sql .f .f90 .f95 .f03 .f08 .pas .pp .dpr .dpk .lpr .inc .dfm .lfm .lpk .sh .bash .json .dm .dme .dmi .dmm .dmf .sln .slnx .csproj .fsproj .vbproj .xaml .razor .cshtml` ‏(`.dm`/`.dme` דורש `uv tool install graphifyy[dm]`; ‏CUDA ‏`.cu`/`.cuh` ו-Metal ‏`.metal` משתמשים בדקדוק C++) | +| Salesforce Apex | `.cls .trigger` (מבוסס regex; מחלקות, ממשקים, enums, מתודות, טריגרים, קשתות SOQL/DML) | +| Terraform / HCL | `.tf .tfvars .hcl` (דורש `uv tool install graphifyy[terraform]`) | +| תצורות MCP | `.mcp.json` ‏`mcp.json` ‏`mcp_servers.json` ‏`claude_desktop_config.json` — מחלץ צומתי שרתים, הפניות לחבילות, דרישות משתני סביבה | +| מניפסטים של חבילות | `apm.yml` ‏`pyproject.toml` ‏`go.mod` ‏`pom.xml` — צומת חבילה קנוני אחד לכל חבילה (לפי שם) בתוספת קשתות `depends_on`, כך שחבילה שמופנית מכמה מניפסטים היא מוקד (hub) יחיד | +| מסמכים | `.md .mdx .qmd .html .txt .rst .yaml .yml` (קישורי markdown ‏`[text](./other.md)` ו-`[[wikilinks]]` הופכים לקשתות `references` בין מסמכים) | +| Office | `.docx .xlsx` (דורש `uv tool install graphifyy[office]`) | +| Google Workspace | `.gdoc .gsheet .gslides` ‏(opt-in; דורש אימות `gws` ו-`--google-workspace`; ‏Sheets דורש `uv tool install graphifyy[google]`) | +| PDF | `.pdf` | +| תמונות | `.png .jpg .webp .gif` | +| וידאו / אודיו | `.mp4 .mov .mp3 .wav` ועוד (דורש `uv tool install graphifyy[video]`) | +| YouTube / כתובות URL | כל כתובת וידאו (דורש `uv tool install graphifyy[video]`) | + +קוד מחולץ מקומית ללא קריאות API ‏(AST באמצעות tree-sitter). כל השאר עובר דרך ה-API של מודל עוזר ה-AI שלכם. + +קובצי `.gdoc`‏, `.gsheet` ו-`.gslides` של Google Drive לשולחן העבודה הם קיצורי דרך, לא תוכן מסמכים. כדי לכלול מסמכי Google Docs, ‏Sheets ו-Slides נטיביים בחילוץ headless, התקינו ואמתו את [`gws` CLI](https://github.com/googleworkspace/cli), ואז הריצו: + +
+ +```bash +uv tool install "graphifyy[google]" # נדרש לרינדור טבלאות Google Sheets +gws auth login -s drive +graphify extract ./docs --google-workspace +``` + +
+ +אפשר גם להגדיר `GRAPHIFY_GOOGLE_WORKSPACE=1`. ‏graphify מייצא קיצורי דרך אל `graphify-out/converted/` כקובצי Markdown נלווים, ואז מחלץ אותם. + +--- + +## פקודות נפוצות + +
+ +```bash +/graphify . # בניית גרף לתיקייה הנוכחית +/graphify ./docs --update # חילוץ מחדש של קבצים שהשתנו בלבד +/graphify . --cluster-only # הרצת אשכול מחדש בלי לחלץ מחדש +/graphify . --cluster-only --resolution 1.5 # קהילות מפורטות יותר +/graphify . --cluster-only --exclude-hubs 99 # הסתרת מוקדי-עזר מדירוג צומתי המפתח +/graphify . --no-viz # דילוג על ה-HTML, רק הדוח וה-JSON +/graphify . --wiki # בניית ויקי markdown מהגרף +graphify export callflow-html # ‏HTML ארכיטקטורה/זרימת-קריאות ב-Mermaid (מתחדש אוטומטית בכל commit אם ההוק מותקן) + +/graphify query "מה מחבר את האימות למסד הנתונים?" +/graphify path "UserService" "DatabasePool" +/graphify explain "RateLimiter" + +/graphify add https://arxiv.org/abs/1706.03762 # הבאת מאמר והוספתו +/graphify add # תמלול והוספת סרטון + +graphify hook install # בנייה מחדש אוטומטית בכל commit +graphify merge-graphs a.json b.json # מיזוג שני גרפים + +graphify prs # לוח PR: מצב CI, סטטוס ביקורת, מיפוי worktree +graphify prs 42 # צלילה עמוקה ל-PR ‏#42 עם השפעת גרף +graphify prs --triage # ‏AI מדרג את תור הביקורות שלכם (משתמש ב-backend המוגדר) +graphify prs --conflicts # ‏PRs שחולקים קהילות גרף — סיכון בסדר המיזוג +``` + +
+ +ראו את [רשימת הפקודות המלאה](#רשימת-הפקודות-המלאה) בהמשך. + +--- + +## התעלמות מקבצים + +צרו קובץ `.graphifyignore` בשורש הפרויקט — אותו תחביר כמו `.gitignore`, כולל שלילה עם `!`. + +**‏`.gitignore` נאכף אוטומטית.** ‏graphify קורא את ה-`.gitignore` בכל תיקייה. אם קיים גם `.graphifyignore`, השניים **ממוזגים** — תבניות `.graphifyignore` מוערכות אחרונות, כך שהן גוברות במקרה של התנגשות (כולל שלילות `!`). הוספת `.graphifyignore` רק מחריגה עוד; היא לעולם לא תחזיר קובץ שה-`.gitignore` כבר החריג. תחולת תת-תיקיות עובדת כמו ב-git — קובץ ignore משפיע רק על תת-העץ שלו. + +
+ +``` +# .graphifyignore +node_modules/ +dist/ +*.generated.py + +# לאנדקס רק את src/, להתעלם מכל השאר +* +!src/ +!src/** +``` + +
+ +--- + +## עבודת צוות + +‏`graphify-out/` מיועד להיכנס ל-git כדי שכל חברי הצוות יתחילו עם מפה מוכנה. + +**תוספות מומלצות ל-`.gitignore`:** + +
+ +``` +graphify-out/cost.json # מקומי בלבד +# graphify-out/cache/ # אופציונלי: commit למהירות, דילוג לשמירה על ריפו קטן +``` + +
+ +> ‏`manifest.json` הוא כעת נייד — המפתחות נשמרים כנתיבים יחסיים ומעוגנים מחדש בטעינה, כך שבטוח לבצע לו commit והדבר חוסך בנייה מלאה מחדש ב-checkout הראשון. + +**תהליך העבודה:** +1. אחד מחברי הצוות מריץ `‎/graphify .` ומבצע commit ל-`graphify-out/`. +2. כולם מושכים — העוזר שלהם קורא את הגרף מיד. +3. הריצו `graphify hook install` לבנייה אוטומטית מחדש אחרי כל commit ‏(AST בלבד, ללא עלות API). זה גם מגדיר merge driver של git כך ש-`graph.json` לעולם לא יישאר עם סימוני קונפליקט — שני מפתחים שמבצעים commit במקביל מקבלים מיזוג-איחוד אוטומטי של הגרפים. +4. כשמסמכים או מאמרים משתנים, הריצו `‎/graphify --update` לרענון הצמתים הללו. + +--- + +## שימוש ישיר בגרף + +
+ +```bash +# שאילתת גרף מהטרמינל +graphify query "הצג את זרימת האימות" +graphify query "מה מחבר בין DigestAuth ל-Response?" --graph graphify-out/graph.json + +# חשיפת הגרף כשרת MCP (לגישת כלים חוזרת) +python -m graphify.serve graphify-out/graph.json +python -m graphify.serve --graph graphify-out/graph.json # גם הדגל --graph מתקבל + +# רישום ב-Kimi Code: +kimi mcp add --transport stdio graphify -- python -m graphify.serve graphify-out/graph.json + +# או הגשה על HTTP כך שכל הצוות מצביע על URL אחד (בלי graphify מקומי): +python -m graphify.serve graphify-out/graph.json --transport http --port 8080 +python -m graphify.serve graphify-out/graph.json --transport http --host 0.0.0.0 --api-key "$SECRET" +``` + +
+ +שרת ה-MCP נותן לעוזר שלכם גישה מובנית: `query_graph`‏, `get_node`‏, `get_neighbors`‏, `shortest_path`‏, `list_prs`‏, `get_pr_impact`‏, `triage_prs`. + +### שרת HTTP משותף + +`--transport stdio` (ברירת המחדל) מריץ שרת מקומי אחד לכל מפתח. `--transport http` מגיש את אותם כלים על גבי MCP Streamable HTTP, כך שתהליך משותף אחד יכול לשרת את הגרף לכל הצוות — הלקוחות מכוונים את תצורת ה-MCP של ה-IDE אל `http://:8080/mcp` במקום להריץ graphify מקומית. + +| דגל | ברירת מחדל | תפקיד | +|---|---|---| +| `--transport {stdio,http}` | `stdio` | סוג התעבורה | +| `--host` | `127.0.0.1` | כתובת ההאזנה ל-HTTP (השתמשו ב-`0.0.0.0` לחשיפה מעבר ל-localhost) | +| `--port` | `8080` | פורט ההאזנה | +| `--api-key` | משתנה סביבה `GRAPHIFY_API_KEY` | דרישת `Authorization: Bearer ` (או `X-API-Key`) | +| `--path` | `/mcp` | נתיב העיגון ב-HTTP | +| `--json-response` | כבוי | החזרת JSON רגיל במקום זרמי SSE | +| `--stateless` | כבוי | ללא מצב פר-סשן (לפריסות מאוזנות-עומס / CI) | +| `--session-timeout` | `3600` | ניקוי סשנים לא פעילים אחרי N שניות (`0` מבטל) | + +ברירת המחדל `127.0.0.1` היא loopback בלבד. הגדירו `--host 0.0.0.0` **וגם** `--api-key` יחד כשחושפים על מארח משותף. אפשר להריץ בקונטיינר: + +
+ +```bash +docker build -t graphify . +docker run -p 8080:8080 -v "$(pwd)/graphify-out:/data" graphify \ + /data/graph.json --transport http --host 0.0.0.0 --api-key "$SECRET" +``` + +
+ +> **הערת WSL / לינוקס:** אובונטו מגיעה עם `python3`, לא `python`. השתמשו ב-venv כדי להימנע מהתנגשויות: + +
+ +```bash +python3 -m venv .venv && .venv/bin/pip install "graphifyy[mcp]" +``` + +
+ +--- + +## משתני סביבה + +אלו נדרשים רק ל**חילוץ headless / CI** ‏(`graphify extract`). בהרצה דרך מיומנות `‎/graphify` בתוך ה-IDE, ה-API של המודל מסופק על ידי סשן ה-IDE — לא נדרשים מפתחות נוספים. + +| משתנה | משמש עבור | מתי נדרש | +|---|---|---| +| `ANTHROPIC_API_KEY` | ‏backend של Claude ‏(Anthropic) | `--backend claude` | +| `ANTHROPIC_BASE_URL` | כתובת endpoint תואם-Anthropic ‏(פרוקסי LiteLLM, שערים, ...) | `--backend claude` (ברירת מחדל: `https://api.anthropic.com`) | +| `ANTHROPIC_MODEL` | שם המודל ל-backend של Claude — ל-endpoints מותאמים, השתמשו בשם/כינוי שהשרת שלכם חושף | `--backend claude` (ברירת מחדל: `claude-sonnet-4-6`) | +| `GEMINI_API_KEY` או `GOOGLE_API_KEY` | ‏backend של Google Gemini | `--backend gemini` | +| `OPENAI_API_KEY` | ‏OpenAI או ממשקים תואמי-OpenAI | `--backend openai` (שרתים מקומיים מקבלים כל ערך לא ריק) | +| `OPENAI_BASE_URL` | כתובת שרת תואם-OpenAI ‏(llama.cpp, ‏vLLM, ‏LM Studio, ...) | `--backend openai` (ברירת מחדל: `https://api.openai.com/v1`) | +| `OPENAI_MODEL` | שם המודל ל-backend של OpenAI — לשרתים בהרצה עצמית, השתמשו בשם/כינוי שהשרת חושף (בדקו את endpoint ה-`/v1/models` שלו) | `--backend openai` (ברירת מחדל: `gpt-4.1-mini`) | +| `DEEPSEEK_API_KEY` | ‏backend של DeepSeek | `--backend deepseek` | +| `MOONSHOT_API_KEY` | ‏backend של Kimi Code | `--backend kimi` | +| `OLLAMA_BASE_URL` | כתובת הרצה מקומית של Ollama | `--backend ollama` (ברירת מחדל: `http://localhost:11434`) | +| `OLLAMA_MODEL` | שם מודל Ollama | `--backend ollama` (ברירת מחדל: זיהוי אוטומטי) | +| `GRAPHIFY_OLLAMA_NUM_CTX` | דריסת גודל חלון ה-KV-cache של Ollama | אופציונלי — מותאם אוטומטית כברירת מחדל | +| `GRAPHIFY_OLLAMA_KEEP_ALIVE` | דקות להשארת מודל Ollama טעון | אופציונלי — ‏`0` לפריקה אחרי כל chunk | +| `AZURE_OPENAI_API_KEY` | ‏backend של Azure OpenAI Service | `--backend azure` | +| `AZURE_OPENAI_ENDPOINT` | כתובת ה-endpoint של משאב Azure | `--backend azure` (נדרש יחד עם מפתח ה-API) | +| `AZURE_OPENAI_API_VERSION` | דריסת גרסת ה-API של Azure | אופציונלי — ברירת מחדל `2024-12-01-preview` | +| `AZURE_OPENAI_DEPLOYMENT` או `GRAPHIFY_AZURE_MODEL` | שם הפריסה ב-Azure | אופציונלי — ברירת מחדל `gpt-4o` | +| `AWS_*` / `~/.aws/credentials` | ‏AWS Bedrock — שרשרת אישורים סטנדרטית | `--backend bedrock` (ללא מפתח API, משתמש ב-IAM) | +| `GRAPHIFY_MAX_WORKERS` | מספר threads למקביליות AST | אופציונלי — גם דגל `--max-workers` | +| `GRAPHIFY_MAX_OUTPUT_TOKENS` | העלאת תקרת הפלט לקורפוסים צפופים | אופציונלי — למשל `32768` לקבצים גדולים | +| `GRAPHIFY_API_TIMEOUT` | ‏timeout לקריאה בשניות עבור HTTP, ‏claude-cli ו-Anthropic SDK ‏(ברירת מחדל: 600) | אופציונלי — גם דגל `--api-timeout` | +| `GRAPHIFY_MAX_RETRIES` | מספר ניסיונות חוזרים לבקשה שנחסמה בקצב (429) לפני ויתור (ברירת מחדל: 6; מכבד `Retry-After`) | אופציונלי — העלו למגבלות ארגוניות נוקשות; ‏`0` מבטל | +| `GRAPHIFY_FORCE` | כפיית בנייה מחדש של הגרף גם עם פחות צמתים | אופציונלי — גם דגל `--force` | +| `GRAPHIFY_GOOGLE_WORKSPACE` | הפעלה אוטומטית של ייצוא Google Workspace | אופציונלי — הגדירו `1` | +| `GRAPHIFY_TRIAGE_BACKEND` | ‏backend עבור `graphify prs --triage` | אופציונלי — מזוהה אוטומטית מהמפתחות הזמינים | +| `GRAPHIFY_TRIAGE_MODEL` | דריסת מודל לטריאז' | אופציונלי — למשל `claude-opus-4-7` | +| `GRAPHIFY_QUERY_LOG` | דריסת נתיב יומן השאילתות (ברירת מחדל: `~/.cache/graphify-queries.log`) | אופציונלי — ערך ריק או `/dev/null` להשתקה | +| `GRAPHIFY_QUERY_LOG_DISABLE` | הגדירו `1` לביטול מוחלט של יומן השאילתות | אופציונלי | +| `GRAPHIFY_QUERY_LOG_RESPONSES` | הגדירו `1` לרישום גם של תשובות תת-גרף מלאות (כבוי כברירת מחדל) | אופציונלי | +| `GRAPHIFY_MAX_GRAPH_BYTES` | דריסת תקרת הגודל של graph.json ‏(512 MiB) — למשל `700MB`, ‏`2GB` או בייטים | אופציונלי — שימושי לקורפוסים גדולים מאוד | +| `GRAPHIFY_LLM_TEMPERATURE` | דריסת טמפרטורת ה-LLM לחילוץ סמנטי — למשל `0.7`, או `none` להשמטה | אופציונלי — מושמט אוטומטית למודלי היסק o1/o3/o4/gpt-5 | + +--- + +## פרטיות + +- **קובצי קוד** — מעובדים מקומית עם tree-sitter. שום דבר לא עוזב את המחשב. קורפוס של קוד בלבד אינו דורש מפתח API — ‏`graphify extract` רץ לגמרי offline. +- **וידאו / אודיו** — מתומלל מקומית עם faster-whisper. שום דבר לא עוזב את המחשב. +- **מסמכים, PDF, תמונות** — נשלחים לעוזר ה-AI שלכם לחילוץ סמנטי (דרך מיומנות `‎/graphify`, עם המודל שסשן ה-IDE שלכם מריץ). ‏`graphify extract` בחילוץ headless דורש `GEMINI_API_KEY` / ‏`GOOGLE_API_KEY` ‏(Gemini), ‏`MOONSHOT_API_KEY` ‏(Kimi), ‏`ANTHROPIC_API_KEY` ‏(Claude), ‏`OPENAI_API_KEY` ‏(OpenAI), ‏`DEEPSEEK_API_KEY` ‏(DeepSeek), מופע Ollama רץ (`OLLAMA_BASE_URL`), אישורי AWS דרך שרשרת הספקים הסטנדרטית (Bedrock — ללא מפתח API, משתמש ב-IAM), או קובץ ההרצה `claude` ‏(Claude Code — ללא מפתח API, משתמש במנוי Claude שלכם). הדגל `--dedup-llm` משתמש באותו מפתח. +- **מיקום נתונים (Data residency)** — ‏`graphify extract` מזהה אוטומטית באיזה ספק להשתמש לפי המפתח שמוגדר (עדיפות: Gemini ‏→ Kimi ‏→ Claude ‏→ OpenAI ‏→ DeepSeek ‏→ Azure ‏→ Bedrock ‏→ Ollama). לקוד עם דרישות מיקום נתונים, השתמשו ב-`--backend ollama` (מקומי לחלוטין) או העבירו דגל `--backend` מפורש. ‏Kimi ‏(`MOONSHOT_API_KEY`) מנתב לשרתי Moonshot AI בסין. +- ללא טלמטריה, ללא מעקב שימוש, ללא אנליטיקה. +- **יומן שאילתות** — כל קריאת `graphify query`‏, `graphify path`‏, `graphify explain` ו-`query_graph` של MCP נרשמת ל-`~/.cache/graphify-queries.log` בפורמט JSON Lines ‏(חותמת זמן, שאלה, קורפוס, צמתים שהוחזרו, משך). תשובות תת-גרף מלאות **אינן** נשמרות כברירת מחדל. הגדירו `GRAPHIFY_QUERY_LOG_DISABLE=1` לביטול, או `GRAPHIFY_QUERY_LOG=/dev/null` להשתקה בלי לבטל את המנגנון. + +--- + +## פתרון בעיות + +**‏`graphify: command not found` אחרי ההתקנה** +ה-CLI מותקן אבל תיקיית ה-bin שלו אינה ב-`PATH` של המעטפת. בחרו את התיקון לפי אופן ההתקנה: +- **uv** ‏(`uv tool install graphifyy`): הפקודה מגיעה לתיקיית הכלים של uv ‏(`~/.local/bin`), שהתקנת macOS/zsh טרייה לרוב לא כוללת ב-`PATH`. הריצו `uv tool update-shell` ופתחו טרמינל חדש. (מצאו את התיקייה עם `uv tool dir --bin`.) +- **pipx** ‏(`pipx install graphifyy`): הריצו `pipx ensurepath` ופתחו טרמינל חדש. +- **pip** ‏(`pip install graphifyy`): ‏pip מתקין סקריפטים לתיקיית bin של המשתמש שאולי אינה ב-`PATH` — הוסיפו את `~/Library/Python/3.x/bin` ‏(macOS) או `~/.local/bin` ‏(לינוקס) ל-`PATH` ב-`~/.zshrc`/`~/.bashrc`, או פשוט הריצו `python -m graphify`. + +**‏`uvx graphify …` או `uv tool run graphify …` לא מצליחים לפתור את `graphify`** +חבילת ה-PyPI היא `graphifyy`; ‏`graphify` הוא רק הפקודה שהיא מספקת. ‏`uv tool run` מתייחס למילה הראשונה כשם *חבילה*, מחפש חבילה בשם `graphify` ומדווח `No solution found … no versions of graphify`. ציינו את החבילה במפורש: `uvx --from graphifyy graphify install` (זהה ל-`uv tool run --from graphifyy graphify install`). או התקינו פעם אחת עם `uv tool install graphifyy` וקראו ל-`graphify` ישירות. + +**‏`python -m graphify` עובד אבל פקודת `graphify` לא** +ה-`PATH` של המעטפת לא כולל את תיקיית ה-bin שאליה הותקנה הפקודה. העדיפו `uv tool install` / ‏`pipx install` על פני `pip` רגיל, ואז הריצו `uv tool update-shell` / ‏`pipx ensurepath` ופתחו טרמינל חדש (ראו הערות ההתקנה לעיל). + +**‏`‎/graphify .` גורם ל-"path not recognized" ב-PowerShell** +‏PowerShell מתייחס ל-`/` מוביל כמפריד נתיבים. השתמשו ב-`graphify .` (בלי לוכסן) ב-Windows. + +**לגרף יש פחות צמתים אחרי `--update` או בנייה מחדש** +אם refactor מחק קבצים, הצמתים הישנים נשארים. העבירו `--force` (או הגדירו `GRAPHIFY_FORCE=1`) לדריסה גם כשהבנייה החדשה קטנה יותר. + +**לגרף יש צמתים כפולים לאותה ישות (ghost duplicates)** +כפילויות רפאים (אותו סימבול מופיע פעמיים — פעם מחילוץ AST עם מיקום מקור, ופעם מחילוץ סמנטי בלעדיו) ממוזגות כעת אוטומטית בזמן הבנייה. אם אתם רואים זאת בגרף שנבנה לפני v0.8.33, הריצו חילוץ מלא מחדש לניקוי: + +
+ +```bash +graphify extract . --force +``` + +
+ +**‏Ollama נגמר לו ה-VRAM / חריגה מחלון ההקשר** +חלון ה-KV-cache מותאם אוטומטית אך עשוי להיות גדול מדי ל-GPU שלכם. הקטינו אותו: + +
+ +```bash +GRAPHIFY_OLLAMA_NUM_CTX=8192 graphify extract ./docs --backend ollama --token-budget 4000 +``` + +
+ +**אזהרות `LLM returned invalid JSON` / ‏`Unterminated string`** +תשובת ה-JSON של המודל פגעה במגבלת אסימוני הפלט ונחתכה באמצע מחרוזת. ‏graphify מתאושש אוטומטית (הוא מפצל את ה-chunk ומחלץ מחדש את החצאים, ומסמך יחיד גדול מדי נחתך תחילה בגבולות כותרות/פסקאות כך שהקובץ כולו עדיין מכוסה), כך שהאזהרות רועשות אך אין אובדן נתונים. להפחתת התופעה, העלו את תקרת הפלט או הקטינו את פלט ה-chunk: + +
+ +```bash +GRAPHIFY_MAX_OUTPUT_TOKENS=16384 graphify extract . --mode deep # הרמת התקרה +graphify extract . --mode deep --token-budget 4000 # קלטים קטנים יותר -> פלט קטן יותר +``` + +
+ +עם שער ענן כמו OpenRouter, העדיפו `--backend openai` (הגדירו `OPENAI_BASE_URL`) על פני שכבת ה-Ollama — זה נתיב תואם-OpenAI נקי יותר. אם למודל יש תקרת פלט משלו, הורדת `--token-budget` היא המנוף האמין. + +**‏HTML של הגרף גדול מדי לפתיחה בדפדפן (מעל 5000 צמתים)** +דלגו על יצירת ה-HTML והשתמשו ב-JSON ישירות: + +
+ +```bash +graphify cluster-only ./my-project --no-viz +graphify query "..." +``` + +
+ +**ל-`graph.json` יש סימוני קונפליקט אחרי ששני מפתחים ביצעו commit במקביל** +הריצו `graphify hook install` — הוא מגדיר merge driver של git שממזג-מאחד את `graph.json` אוטומטית כך שקונפליקטים לא קורים בכלל. + +**החילוץ מחזיר צמתים/קשתות ריקים למסמכים או PDF** +מסמכים, PDF ותמונות דורשים קריאת LLM — קורפוסים של קוד בלבד אינם דורשים מפתח. ודאו שמפתח ה-API מוגדר וה-backend נכון: + +
+ +```bash +ANTHROPIC_API_KEY=sk-... graphify extract ./docs --backend claude +``` + +
+ +**אזהרת חוסר התאמה בגרסת המיומנות ב-IDE** +גרסת graphify המותקנת שונה מקובץ המיומנות. עדכנו: + +
+ +```bash +uv tool upgrade graphifyy +graphify install # דורס את קובץ המיומנות +``` + +
+ +--- + +## רשימת הפקודות המלאה + +
+ +``` +/graphify # הרצה על התיקייה הנוכחית +/graphify ./raw # הרצה על תיקייה מסוימת +/graphify ./raw --mode deep # חילוץ קשרים אגרסיבי יותר +/graphify ./raw --update # חילוץ מחדש של קבצים שהשתנו בלבד +/graphify ./raw --directed # שמירת כיוון הקשתות +/graphify ./raw --cluster-only # הרצת אשכול מחדש על גרף קיים +/graphify ./raw --no-viz # דילוג על ויזואליזציית HTML +/graphify ./raw --obsidian # יצירת כספת Obsidian +/graphify ./raw --obsidian --obsidian-dir ~/vault # כתיבה לכספת קיימת (לעולם לא דורס פתקים או תצורת .obsidian שלכם) +/graphify ./raw --wiki # בניית ויקי markdown שסוכנים יכולים לסרוק +/graphify ./raw --svg # ייצוא graph.svg +/graphify ./raw --graphml # ייצוא ל-Gephi / yEd +/graphify ./raw --neo4j # יצירת cypher.txt ל-Neo4j +/graphify ./raw --neo4j-push bolt://localhost:7687 +/graphify ./raw --falkordb # יצירת cypher.txt ל-FalkorDB +/graphify ./raw --falkordb-push falkordb://localhost:6379 +/graphify ./raw --watch # סנכרון אוטומטי כשקבצים משתנים +/graphify ./raw --mcp # הפעלת שרת MCP stdio + +/graphify add https://arxiv.org/abs/1706.03762 +/graphify add +/graphify add https://... --author "Name" --contributor "Name" + +/graphify query "מה מחבר את ה-attention לאופטימייזר?" +/graphify query "..." --dfs --budget 1500 +/graphify path "DigestAuth" "Response" +/graphify explain "SwinTransformer" + +graphify save-result --question "Q" --answer "A" --nodes Foo Bar --outcome useful # תיעוד תוצאת שאלה-תשובה (זיכרון עבודה; outcome ∈ useful|dead_end|corrected) +graphify reflect # איחוד תוצאות graphify-out/memory/ אל reflections/LESSONS.md +graphify reflect --if-stale # לא עושה דבר אם LESSONS.md כבר חדש מכל הקלטים (זול להרצה בכל סשן) +graphify reflect --out docs/LESSONS.md # כתיבת מסמך הלקחים למקום אחר +graphify reflect --graph graphify-out/graph.json # קיבוץ לקחים לפי קהילה + כתיבת שכבת זיכרון העבודה (.graphify_learning.json) + # השכבה מתייגת צמתים preferred/tentative/contested (משוקלל-עדכניות, עם מקור); + # graphify explain / query מציגים אז רמז "Lesson:", מסומן "code changed — re-verify" כשהמקור התקדם + +graphify uninstall # הסרה מכל הפלטפורמות במכה אחת +graphify uninstall --purge # מחיקה גם של graphify-out/ +graphify uninstall --project --platform codex # הסרת קובצי התקנה ברמת פרויקט בלבד + +graphify hook install # הוקים של post-commit + post-checkout +graphify hook uninstall +graphify hook status + +# הנחיות עוזר קבועות - פר פלטפורמה +graphify claude install # CLAUDE.md + הוק PreToolUse ‏(Claude Code) +graphify claude uninstall +graphify codebuddy install # CODEBUDDY.md + הוק PreToolUse ‏(CodeBuddy) +graphify codebuddy uninstall +graphify codex install # AGENTS.md + הוק PreToolUse ב-.codex/hooks.json ‏(Codex) +graphify opencode install # AGENTS.md + תוסף tool.execute.before ‏(OpenCode) +graphify kilo install # מיומנות Kilo נטיבית + פקודת /graphify + ‏AGENTS.md + תוסף .kilo +graphify kilo uninstall +graphify cursor install # .cursor/rules/graphify.mdc ‏(Cursor) +graphify cursor uninstall +graphify gemini install # GEMINI.md + הוק BeforeTool ‏(Gemini CLI) +graphify gemini uninstall +graphify copilot install # קובץ מיומנות (GitHub Copilot CLI) +graphify copilot uninstall +graphify aider install # AGENTS.md ‏(Aider) +graphify aider uninstall +graphify claw install # AGENTS.md ‏(OpenClaw) +graphify claw uninstall +graphify droid install # AGENTS.md ‏(Factory Droid) +graphify droid uninstall +graphify trae install # AGENTS.md ‏(Trae) +graphify trae uninstall +graphify trae-cn install # AGENTS.md ‏(Trae CN) +graphify trae-cn uninstall +graphify hermes install # AGENTS.md + ~/.hermes/skills/ ‏(Hermes) +graphify hermes uninstall +graphify amp install # קובץ מיומנות (Amp) +graphify amp uninstall +graphify agents install # ~/.agents/skills/ + AGENTS.md ‏(חוצה-פלטפורמות; כינוי: graphify skills) +graphify agents uninstall +graphify kiro install # .kiro/skills/ + .kiro/steering/graphify.md ‏(Kiro IDE/CLI) +graphify kiro uninstall +graphify pi install # קובץ מיומנות (Pi coding agent) +graphify pi uninstall +graphify devin install # קובץ מיומנות + .windsurf/rules/graphify.md ‏(Devin CLI) +graphify devin uninstall +graphify antigravity install # .agents/rules + .agents/workflows ‏(Google Antigravity) +graphify antigravity uninstall + +graphify extract ./docs # חילוץ LLM ‏headless ל-CI ‏(ללא IDE) +graphify extract ./docs --backend gemini # ‏backend מפורש: gemini, kimi, claude, openai, deepseek, ollama, bedrock או claude-cli +graphify extract ./docs --backend gemini --model gemini-3.1-pro-preview +graphify extract ./docs --backend ollama # ‏Ollama מקומי (הגדירו OLLAMA_BASE_URL / OLLAMA_MODEL) - ללא מפתח API ל-loopback +OPENAI_BASE_URL=http://localhost:8080/v1 OPENAI_MODEL=my-model graphify extract ./docs --backend openai # כל שרת תואם-OpenAI ‏(llama.cpp, vLLM, LM Studio) +ANTHROPIC_BASE_URL=http://localhost:4000 ANTHROPIC_MODEL=my-model graphify extract ./docs --backend claude # כל endpoint תואם-Anthropic ‏(פרוקסי LiteLLM, שערים) +GRAPHIFY_OLLAMA_NUM_CTX=32768 graphify extract ./docs --backend ollama # דריסת חלון ה-KV-cache (מותאם אוטומטית כברירת מחדל) +GRAPHIFY_OLLAMA_KEEP_ALIVE=0 graphify extract ./docs --backend ollama # פריקת המודל אחרי כל chunk (חוסך VRAM ב-GPU קטן) +graphify extract ./docs --backend bedrock # AWS Bedrock דרך IAM - ללא מפתח API, שרשרת אישורי AWS +graphify extract ./docs --backend claude-cli # ניתוב דרך Claude Code CLI - ללא מפתח API, דרך מנוי Claude שלכם +graphify extract ./docs --backend azure # Azure OpenAI (הגדירו AZURE_OPENAI_API_KEY + AZURE_OPENAI_ENDPOINT) +graphify extract ./docs --max-workers 16 # מקביליות AST ‏(גם GRAPHIFY_MAX_WORKERS) +graphify extract --postgres "postgresql://user:pass@host/db" # אינטרוספקציה ישירה של סכמת PostgreSQL חיה +graphify extract ./my-workspace --cargo # אינטרוספקציה ישירה של תלויות Cargo workspace ברוסט +graphify extract ./docs --token-budget 30000 # ‏chunks סמנטיים קטנים יותר למודלים מקומיים/קטנים +graphify extract ./docs --max-concurrency 2 # פחות קריאות LLM מקביליות (שימושי להרצה מקומית) +graphify extract ./docs --api-timeout 900 # ‏timeout ארוך יותר למודלים מקומיים איטיים (ברירת מחדל 600 שניות) +graphify extract ./docs --google-workspace # ייצוא .gdoc/.gsheet/.gslides דרך gws לפני החילוץ +graphify extract ./docs --mode deep # חילוץ סמנטי עשיר יותר עם system prompt מורחב +graphify extract ./docs --no-cluster # חילוץ גולמי בלבד, דילוג על אשכול +graphify extract ./docs --timing # הדפסת זמני ריצה פר-שלב ל-stderr ‏(עובד גם ב-cluster-only) +graphify extract ./docs --force # דריסת graph.json גם אם לגרף החדש פחות צמתים (אחרי refactors או לניקוי כפילויות רפאים) +graphify extract ./docs --dedup-llm # ‏LLM כמכריע לזוגות ישויות עמומים (משתמש באותו מפתח API) +graphify extract ./docs --global --as myrepo # חילוץ ורישום בגרף הגלובלי חוצה-הפרויקטים +GRAPHIFY_MAX_OUTPUT_TOKENS=32768 graphify extract ./docs --backend claude # הרמת תקרת הפלט לקורפוסים צפופים + +graphify export callflow-html # graphify-out/-callflow.html +graphify export callflow-html --max-sections 8 # הגבלת מספר קטעי הארכיטקטורה שנוצרים +graphify export callflow-html --output docs/arch.html +graphify export callflow-html ./some-repo/graphify-out + +graphify global add graphify-out/graph.json --as myrepo # רישום גרף פרויקט אל ~/.graphify/global-graph.json +graphify global remove myrepo # הסרת פרויקט מהגרף הגלובלי +graphify global list # הצגת כל המאגרים הרשומים + ספירות צמתים/קשתות +graphify global path # הדפסת הנתיב לקובץ הגרף הגלובלי + +graphify prs # לוח PR: ‏CI, ביקורת, worktree, השפעת גרף +graphify prs 42 # צלילה עמוקה ל-PR ‏#42 +graphify prs --triage # דירוג טריאז' AI ‏(מזהה backend אוטומטית מהסביבה) +graphify prs --worktrees # מיפוי worktree ← branch ← PR +graphify prs --conflicts # ‏PRs שחולקים קהילות גרף (סיכון בסדר המיזוג) +graphify prs --base main # סינון ל-PRs שמכוונים ל-branch בסיס מסוים +graphify prs --repo owner/repo # הרצה מול מאגר GitHub אחר +GRAPHIFY_TRIAGE_BACKEND=kimi graphify prs --triage # ‏backend מסוים לטריאז' + +graphify clone https://github.com/karpathy/nanoGPT +graphify merge-graphs a.json b.json --out merged.json +graphify --version # הדפסת הגרסה המותקנת +graphify watch ./src +graphify check-update ./src +graphify update ./src +graphify update ./src --no-cluster # דילוג על אשכול מחדש, כתיבת גרף AST גולמי בלבד +graphify update ./src --force # דריסה גם אם לגרף החדש פחות צמתים +graphify cluster-only ./my-project +graphify cluster-only ./my-project --graph path/to/graph.json # מיקום גרף מותאם +graphify cluster-only ./my-project --max-concurrency 16 --batch-size 200 # תיוג קהילות מקבילי (גרפים גדולים) +graphify cluster-only ./my-project --resolution 1.5 # יותר קהילות, קטנות יותר +graphify cluster-only ./my-project --exclude-hubs 99 # החרגת צומתי p99 מהחלוקה +graphify cluster-only ./my-project --no-label # השארת מצייני "Community N" +graphify cluster-only ./my-project --backend=gemini # ‏backend לשמות הקהילות +graphify cluster-only ./my-project --backend=gemini --model gemini-2.5-pro # מודל מסוים +graphify label ./my-project # מתן שמות (מחדש) לקהילות עם ה-backend המוגדר +graphify label ./my-project --backend=openai --model gpt-4o # כפיית backend ומודל מסוימים +``` + +
+ +> **שמות קהילות:** בתוך סוכן (Claude Code, ‏Gemini CLI) הסוכן נותן שמות לקהילות בעצמו. בהרצת ה-CLI החשוף, `cluster-only` נותן שמות אוטומטית עם ה-backend המוגדר (מובנה או ספק תואם-OpenAI מותאם) — העבירו `--no-label` להשארת `Community N`, או הריצו `graphify label` ליצירת שמות מחדש לפי דרישה. + +--- + +## ללמוד עוד + +- [איך זה עובד](../how-it-works.md) — צינור החילוץ, זיהוי קהילות, ניקוד ביטחון, מדדים +- [ARCHITECTURE.md](../../ARCHITECTURE.md) — פירוק מודולים, איך מוסיפים שפה +- [אינטגרציות אופציונליות](../docker-mcp-sqlite.md) — ‏Docker MCP Toolkit + ‏SQLite + +--- + +## נבנה על graphify — ‏Penpax + +[**Penpax**](https://graphify.com) היא השכבה התמידית שנבנתה מעל graphify — היא מיישמת את אותה גישת גרף על כל חיי העבודה שלכם: פגישות, היסטוריית דפדפן, מיילים, קבצים וקוד, ומתעדכנת ברציפות ברקע. + +נבנתה לאנשים שהעבודה שלהם מפוזרת על פני מאות שיחות ומסמכים שהם לעולם לא יוכלו לשחזר במלואם. ללא ענן, לגמרי על המכשיר. + +**גרסת ניסיון חינמית תושק בקרוב.** [הצטרפו לרשימת ההמתנה ←](https://graphify.com) + +--- + +
+תרומה לפרויקט + +### הקמת סביבת פיתוח + +הפרויקט משתמש ב-[uv](https://docs.astral.sh/uv/) לתהליך הפיתוח. התקינו אותו פעם אחת, ואז: + +
+ +```bash +git clone https://github.com/safishamsi/graphify.git +cd graphify +git checkout v8 # ‏branch הפיתוח הפעיל + +# יצירת venv לפרויקט והתקנת graphify + כל התוספים + קבוצת dev +# ‏(pytest). ‏uv מתקין את קבוצת התלויות dev כברירת מחדל; העבירו --no-dev לדילוג. +uv sync --all-extras +``` + +
+ +אימות ההתקנה במצב עריכה: + +
+ +```bash +uv run graphify --version +uv run python -c "import graphify; print(graphify.__file__)" +``` + +
+ +### הרצת בדיקות + +
+ +```bash +uv run pytest tests/ -q # הרצת החבילה המלאה +uv run pytest tests/test_extract.py -q # מודול אחד +uv run pytest tests/ -q -k "python" # סינון לפי שם +``` + +
+ +> הערת macOS: חבילת הבדיקות כוללת גם `sample.f90` וגם `sample.F90`. אלו מתנגשים במערכות קבצים HFS+ / APFS שאינן רגישות לרישיות. הריצו בלינוקס או בקונטיינר Docker אם צריך לבדוק את שתי גרסאות ה-Fortran יחד. + +### תהליך עבודה ב-Git + +- הפיתוח הפעיל מתרחש ב-branch ‏`v8`. +- סגנון commit: ‏`fix: ` / ‏`feat: ` / ‏`docs: ` +- לפני פתיחת PR, הריצו `uv run pytest tests/ -q` וודאו שהוא עובר. +- הוסיפו קובץ fixture ל-`tests/fixtures/` ובדיקות ל-`tests/test_languages.py` לכל מחלץ שפה חדש. + +### מה כדאי לתרום + +**דוגמאות עבודה (worked examples)** הן התרומה השימושית ביותר. הריצו `‎/graphify` על קורפוס אמיתי, שמרו את הפלט ב-`worked/{slug}/`, כתבו `review.md` כן שמכסה מה הגרף קלע ומה פספס, ופתחו PR. + +**באגים בחילוץ** — פתחו issue עם קובץ הקלט, רשומת ה-cache ‏(`graphify-out/cache/`) ומה חסר או שגוי. + +ראו [ARCHITECTURE.md](../../ARCHITECTURE.md) לאחריות המודולים ואיך מוסיפים שפה. + +
diff --git a/docs/translations/README.uk-UA.md b/docs/translations/README.uk-UA.md index 337a3c465..2fa632544 100644 --- a/docs/translations/README.uk-UA.md +++ b/docs/translations/README.uk-UA.md @@ -1,5 +1,5 @@

- Graphify + Graphify

@@ -529,11 +529,11 @@ graphify cluster-only ./my-project --exclude-hubs 99 # виключи ## Побудовано на graphify — Penpax -[**Penpax**](https://graphifylabs.ai) — це завжди активний шар поверх graphify, він застосовує той самий графовий підхід до всього робочого життя: зустрічей, історії браузера, email-ів, файлів і коду, постійно оновлюючись у фоновому режимі. +[**Penpax**](https://graphify.com) — це завжди активний шар поверх graphify, він застосовує той самий графовий підхід до всього робочого життя: зустрічей, історії браузера, email-ів, файлів і коду, постійно оновлюючись у фоновому режимі. Створений для людей, чия робота розкидана по сотнях розмов і документів, які неможливо повністю відтворити. Без хмари, повністю на пристрої. -**Безкоштовна пробна версія незабаром.** [Приєднайтесь до списку очікування →](https://graphifylabs.ai) +**Безкоштовна пробна версія незабаром.** [Приєднайтесь до списку очікування →](https://graphify.com) --- diff --git a/graphify/__main__.py b/graphify/__main__.py index e620d97a5..924ae986d 100644 --- a/graphify/__main__.py +++ b/graphify/__main__.py @@ -1,6 +1,7 @@ """graphify CLI - `graphify install` sets up the Claude Code skill.""" from __future__ import annotations +import errno import functools import json import os @@ -23,30 +24,114 @@ # same override (#1423). from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT +# Install/uninstall subsystem moved to graphify/install.py; re-exported here so +# `from graphify.__main__ import ` keeps working unchanged. +from graphify.install import ( # noqa: E402,F401 + dispatch_install_cli, + _agents_install, + _agents_platform_install, + _agents_platform_uninstall, + _agents_uninstall, + _always_on, + _amp_install, + _amp_legacy_cleanup, + _amp_uninstall, + _antigravity_finalize, + _antigravity_install, + _antigravity_uninstall, + _canonical_platform, + _claude_pretooluse_hooks, + _copy_skill_file, + _cursor_install, + _cursor_uninstall, + _devin_rules_install, + _devin_rules_uninstall, + _gemini_hook, + _install_claude_hook, + _install_codebuddy_hook, + _install_codex_hook, + _install_gemini_hook, + _install_kilo_plugin, + _install_opencode_plugin, + _install_skill_references, + _kilo_config_path, + _kilo_config_write_path, + _kilo_install, + _kilo_uninstall, + _kilo_uninstall_global, + _kiro_install, + _kiro_uninstall, + _load_json_like, + _packaged_skill_refs_dir, + _platform_skill_destination, + _print_banner, + _print_install_usage, + _print_project_git_add_hint, + _project_install, + _project_scope_root, + _project_uninstall, + _project_uninstall_all, + _refresh_all_version_stamps, + _remove_claude_skill_registration, + _remove_skill_file, + _replace_or_append_section, + _resolve_graphify_exe, + _skill_registration, + _strip_graphify_hook, + _strip_graphify_md_section, + _strip_json_comments, + _uninstall_claude_hook, + _uninstall_codebuddy_hook, + _uninstall_codex_hook, + _uninstall_gemini_hook, + _uninstall_kilo_plugin, + _uninstall_opencode_plugin, + claude_install, + claude_uninstall, + codebuddy_install, + codebuddy_uninstall, + gemini_install, + gemini_uninstall, + install, + uninstall_all, + vscode_install, + vscode_uninstall, + _PLATFORM_ALIASES, + _CLAUDE_MD_MARKER, + _CODEBUDDY_MD_MARKER, + _AGENTS_MD_MARKER, + _GEMINI_MD_MARKER, + _VSCODE_INSTRUCTIONS_MARKER, + _ANTIGRAVITY_RULES_PATH, + _ANTIGRAVITY_WORKFLOW_PATH, + _ANTIGRAVITY_WORKFLOW, + _CURSOR_RULE_PATH, + _CURSOR_RULE, + _DEVIN_RULES_PATH, + _DEVIN_RULES, + _KILO_PLUGIN_JS, + _KILO_PLUGIN_PATH, + _KILO_CONFIG_JSON_PATH, + _KILO_CONFIG_JSONC_PATH, + _OPENCODE_PLUGIN_JS, + _OPENCODE_PLUGIN_PATH, + _OPENCODE_CONFIG_PATH, + _PLATFORM_CONFIG, +) +from graphify.cli import ( # noqa: E402,F401 + dispatch_command, + _StageTimer, + _clone_repo, + _default_graph_path, + _enforce_graph_size_cap_or_exit, + _run_hook_guard, + _SEARCH_NUDGE, + _READ_NUDGE, + _HOOK_SOURCE_EXTS, + _GEMINI_NUDGE_TEXT, +) -@functools.lru_cache(maxsize=None) -def _always_on(basename: str) -> str: - """Read a packaged always-on instruction block from graphify/always_on/. - The six always-on blocks (CLAUDE.md / AGENTS.md / GEMINI.md / VS Code - Copilot instructions / Antigravity rules / Kiro steering) live as committed - markdown next to this module, generated by tools/skillgen from a single - human-edited fragment and guarded against drift by ``skillgen --check``. The - installer injects them verbatim via ``_replace_or_append_section``, so the - bytes here must match the former triple-quoted constant exactly — the - always-on-roundtrip validator proves that. - """ - path = Path(__file__).parent / "always_on" / f"{basename}.md" - try: - return path.read_text(encoding="utf-8") - except OSError as exc: - # Defer to use-time so a missing/corrupt packaged block can't crash module - # import (which would brick every CLI command, not just install). Reached - # only by an install/integration path that actually needs this block. - raise RuntimeError( - f"graphify install is incomplete: missing always-on block '{basename}' " - f"at {path}. Reinstall graphifyy (e.g. `uv tool install --reinstall graphifyy`)." - ) from exc _ALWAYS_ON_ALIASES = { @@ -70,53 +155,10 @@ def __getattr__(name: str) -> str: raise AttributeError(f"module {__name__!r} has no attribute {name!r}") -def _default_graph_path() -> str: - return str(Path(_GRAPHIFY_OUT) / "graph.json") -class _StageTimer: - """Print per-stage wall-clock timings to stderr when --timing is set (#1490). - Monotonic (perf_counter), diagnostic-only: emits ``[graphify timing] : - N.Ns`` after each stage and a final total. Off by default, so normal output is - byte-identical and machine-read stdout is untouched. - """ - def __init__(self, enabled: bool) -> None: - import time as _time - self._now = _time.perf_counter - self.enabled = enabled - self.start = self._now() - self._last = self.start - - def mark(self, stage: str) -> None: - now = self._now() - if self.enabled: - print(f"[graphify timing] {stage}: {now - self._last:.1f}s", file=sys.stderr) - self._last = now - - def total(self) -> None: - if self.enabled: - print(f"[graphify timing] total: {self._now() - self.start:.1f}s", file=sys.stderr) - - -def _enforce_graph_size_cap_or_exit(gp: Path) -> None: - """Reject oversized graph files before parsing (CLI exit-on-fail flavor). - - Delegates to ``graphify.security.check_graph_file_size_cap`` and turns the - raised ``ValueError`` into a CLI-style ``error: ...`` message + exit 1. - Use this from ``__main__.py`` subcommands that already use the ``print + - sys.exit(1)`` idiom. Library/MCP/loader callers (``serve._load_graph``, - ``build``, ``benchmark``, ``tree_html``, ``callflow_html``, ``prs``, - ``global_graph``, ``watch``, ``export``) call the security helper directly - and let the ``ValueError`` propagate. - """ - from graphify.security import check_graph_file_size_cap - try: - check_graph_file_size_cap(gp) - except ValueError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) def _check_skill_version(skill_dst: Path) -> None: @@ -132,7 +174,7 @@ def _check_skill_version(skill_dst: Path) -> None: except OSError: return if not skill_exists: - print(" warning: skill dir exists but SKILL.md is missing. Run 'graphify install' to repair.") + print(" warning: skill dir exists but SKILL.md is missing. Run 'graphify install' to repair.", file=sys.stderr) return # A progressive SKILL.md links to its references/ sidecar. If the body points # at references/ but the dir is gone (manual delete, partial upgrade), the @@ -185,641 +227,54 @@ def _version_tuple(version: str) -> tuple[int, ...]: return tuple(parts) -def _refresh_all_version_stamps() -> None: - """After a successful install, update .graphify_version in all other known skill dirs. - Prevents stale-version warnings from platforms that were installed previously - but not explicitly re-installed during this upgrade. - """ - for name in _PLATFORM_CONFIG: - skill_dst = _platform_skill_destination(name) - vf = skill_dst.parent / ".graphify_version" - if skill_dst.exists(): - vf.write_text(__version__, encoding="utf-8") - - -def _platform_skill_destination(platform_name: str, *, project: bool = False, project_dir: Path | None = None) -> Path: - """Return the skill destination for a platform and scope.""" - if platform_name == "gemini": - if project: - return (project_dir or Path(".")) / ".gemini" / "skills" / "graphify" / "SKILL.md" - if platform.system() == "Windows": - return Path.home() / ".agents" / "skills" / "graphify" / "SKILL.md" - return Path.home() / ".gemini" / "skills" / "graphify" / "SKILL.md" - - if platform_name == "opencode": - if project: - return (project_dir or Path(".")) / ".opencode" / "skills" / "graphify" / "SKILL.md" - return Path.home() / ".config" / "opencode" / "skills" / "graphify" / "SKILL.md" - - if platform_name == "hermes": - if project: - return (project_dir or Path(".")) / ".hermes" / "skills" / "graphify" / "SKILL.md" - # On Windows, Hermes scans %LOCALAPPDATA%\hermes\skills, not ~/.hermes (#1403). - if platform.system() == "Windows": - local_appdata = Path(os.environ.get("LOCALAPPDATA") or (Path.home() / "AppData" / "Local")) - return local_appdata / "hermes" / "skills" / "graphify" / "SKILL.md" - return Path.home() / ".hermes" / "skills" / "graphify" / "SKILL.md" - - if platform_name == "devin": - if project: - return (project_dir or Path(".")) / ".devin" / "skills" / "graphify" / "SKILL.md" - return Path.home() / ".config" / "devin" / "skills" / "graphify" / "SKILL.md" - - if platform_name == "amp": - if project: - return (project_dir or Path(".")) / ".agents" / "skills" / "graphify" / "SKILL.md" - return Path.home() / ".config" / "agents" / "skills" / "graphify" / "SKILL.md" - - if platform_name == "agents": - # The generic Agent-Skills target: project ./.agents/skills, global the - # spec's user-global ~/.agents/skills (read by `npx skills` and compliant - # frameworks), NOT amp's ~/.config/agents/skills. - if project: - return (project_dir or Path(".")) / ".agents" / "skills" / "graphify" / "SKILL.md" - return Path.home() / ".agents" / "skills" / "graphify" / "SKILL.md" - - if platform_name in ("antigravity", "antigravity-windows"): - if project: - return (project_dir or Path(".")) / ".agents" / "skills" / "graphify" / "SKILL.md" - # Global Antigravity skill dir (all workspaces): ~/.gemini/config/skills/ - return Path.home() / ".gemini" / "config" / "skills" / "graphify" / "SKILL.md" - - cfg = _PLATFORM_CONFIG[platform_name] - if project: - return (project_dir or Path(".")) / cfg["skill_dst"] - - if platform_name in ("claude", "windows") and os.environ.get("CLAUDE_CONFIG_DIR"): - return Path(os.environ["CLAUDE_CONFIG_DIR"]) / "skills" / "graphify" / "SKILL.md" - return Path.home() / cfg["skill_dst"] - - -def _packaged_skill_refs_dir(platform_name: str) -> Path | None: - """Return the packaged references source dir for a progressive platform, else None. - - A platform opts into progressive disclosure by setting ``skill_refs`` in its - ``_PLATFORM_CONFIG`` entry. The value names a bundle under - ``graphify/skills//references/``. Reuse keys (e.g. trae-cn) point at - their twin's bundle. - - ``gemini`` has no ``_PLATFORM_CONFIG`` entry: it installs claude's - ``skill.md`` body verbatim (see ``_copy_skill_file``). Since that body is the - lean progressive core that links to ``references/``, gemini needs claude's - references/ sidecar too, or its SKILL.md ships with dead pointers. So gemini - resolves to the claude bundle rather than opting out. - - Bundles ship one platform-group at a time. A host whose bundle directory - ``graphify/skills//`` is not in this build has not gone progressive - yet, so this returns None and the host installs today's monolithic SKILL.md - with no references/ sidecar. Only when the bundle directory IS present does - this return the references path; if that directory then lacks its - ``references/`` subdir, ``_copy_skill_file`` hard-fails (a malformed bundle, - the empty-sidecar regression the wheel-content test also guards). - """ - if platform_name == "gemini": - bundle = "claude" - else: - bundle = _PLATFORM_CONFIG[platform_name].get("skill_refs") - if not bundle: - return None - bundle_dir = Path(__file__).parent / "skills" / bundle - if not bundle_dir.is_dir(): - return None - return bundle_dir / "references" - - -def _install_skill_references(skill_dst: Path, refs_src: Path) -> None: - """Atomically install a packaged references/ sidecar next to SKILL.md. - - Stages the packaged dir into ``references.tmp`` (copytree), drops any stale - ``references/`` already on disk, then ``os.replace``-renames the staged dir - into place. The rename is atomic on the same filesystem, so an interrupted - install never leaves a half-written references/ visible to the agent. - """ - refs_dst = skill_dst.parent / "references" - refs_staged = skill_dst.parent / "references.tmp" - if refs_staged.exists(): - shutil.rmtree(refs_staged) - try: - shutil.copytree(refs_src, refs_staged) - if refs_dst.exists(): - shutil.rmtree(refs_dst) - os.replace(refs_staged, refs_dst) - except Exception: - if refs_staged.exists(): - shutil.rmtree(refs_staged, ignore_errors=True) - raise -def _copy_skill_file(platform_name: str, *, project: bool = False, project_dir: Path | None = None) -> Path: - """Copy a packaged skill file and write its version stamp. - For progressive platforms (those with ``skill_refs`` set), the packaged - ``references/`` sidecar is installed alongside SKILL.md and the single - ``.graphify_version`` stamp covers both. For monolith platforms (no - ``skill_refs``), any orphan ``references/`` left by a prior progressive - install is removed so the on-disk layout matches the package. - """ - skill_file = "skill.md" if platform_name == "gemini" else _PLATFORM_CONFIG[platform_name]["skill_file"] - skill_src = Path(__file__).parent / skill_file - if not skill_src.exists(): - print(f"error: {skill_file} not found in package - reinstall graphify", file=sys.stderr) - sys.exit(1) - - refs_src = _packaged_skill_refs_dir(platform_name) - if refs_src is not None and not refs_src.exists(): - # Progressive platform declared a references bundle that is missing from - # the package. Fail loud rather than silently shipping an empty sidecar. - print( - f"error: references for '{platform_name}' not found in package " - f"({refs_src}) - reinstall graphify", - file=sys.stderr, - ) - sys.exit(1) - - skill_dst = _platform_skill_destination(platform_name, project=project, project_dir=project_dir) - skill_dst.parent.mkdir(parents=True, exist_ok=True) - - # Install the references/ sidecar (or clear an orphan one) BEFORE writing - # SKILL.md, so SKILL.md is the last artifact laid down. An install that is - # interrupted partway then leaves no SKILL.md rather than a SKILL.md that - # points at an absent references/ dir. - if refs_src is not None: - _install_skill_references(skill_dst, refs_src) - print(f" references -> {skill_dst.parent / 'references'}") - else: - # Monolith (or progressive-with-no-refs): clear any orphan references/. - orphan_refs = skill_dst.parent / "references" - if orphan_refs.exists(): - shutil.rmtree(orphan_refs) - - # SKILL.md last (crash-safety), via an atomic temp + rename. - tmp_dst = skill_dst.with_suffix(skill_dst.suffix + ".tmp") - try: - shutil.copy(skill_src, tmp_dst) - os.replace(tmp_dst, skill_dst) - except Exception: - try: - tmp_dst.unlink(missing_ok=True) - except OSError: - pass - raise - - (skill_dst.parent / ".graphify_version").write_text(__version__, encoding="utf-8") - print(f" skill installed -> {skill_dst}") - return skill_dst - - -def _remove_skill_file(platform_name: str, *, project: bool = False, project_dir: Path | None = None) -> bool: - """Remove a platform skill file and its version stamp without touching other scopes.""" - skill_dst = _platform_skill_destination(platform_name, project=project, project_dir=project_dir) - removed = False - if skill_dst.exists(): - skill_dst.unlink() - print(f" skill removed -> {skill_dst}") - removed = True - version_file = skill_dst.parent / ".graphify_version" - if version_file.exists(): - version_file.unlink() - removed = True - refs_dir = skill_dst.parent / "references" - if refs_dir.exists(): - shutil.rmtree(refs_dir) - removed = True - for d in (skill_dst.parent, skill_dst.parent.parent, skill_dst.parent.parent.parent): - try: - d.rmdir() - except OSError: - break - return removed - - -def _project_scope_root(path: Path, project_dir: Path) -> Path: - """Return the top-level project artifact for a project-scoped skill path.""" - try: - rel = path.relative_to(project_dir) - except ValueError: - return path - return project_dir / rel.parts[0] if rel.parts else path -def _remove_claude_skill_registration(project_dir: Path) -> None: - """Remove the project-scoped Claude skill registration file/section.""" - claude_md = project_dir / ".claude" / "CLAUDE.md" - if not claude_md.exists(): - return - content = claude_md.read_text(encoding="utf-8") - if "# graphify" not in content: - return - cleaned = re.sub(r"\n*# graphify\n.*?(?=\n# |\Z)", "", content, flags=re.DOTALL).rstrip() - if cleaned: - claude_md.write_text(cleaned + "\n", encoding="utf-8") - print(f" CLAUDE.md -> graphify skill registration removed from {claude_md}") - else: - claude_md.unlink() - print(f" CLAUDE.md -> deleted {claude_md}") - - -def _print_project_git_add_hint(paths: list[Path]) -> None: - unique: list[str] = [] - for path in paths: - text = path.as_posix().rstrip("/") - if path.exists() and path.is_dir(): - text += "/" - if text not in unique: - unique.append(text) - if not unique: - return - print() - print("Project-scoped install. Add to version control:") - print(f" git add {' '.join(unique)}") - -_SETTINGS_HOOK = { - # Claude Code v2.1.117+ removed dedicated Grep/Glob tools; searches now go through Bash. - # We match on Bash and inspect the command string to avoid firing on every shell call. - "matcher": "Bash", - "hooks": [ - { - "type": "command", - "command": ( - "CMD=$(python3 -c \"" - "import json,sys; d=json.load(sys.stdin); " - "print(d.get('tool_input',d).get('command',''))\" 2>/dev/null || true); " - "case \"$CMD\" in " - r"*grep*|*rg\ *|*ripgrep*|*find\ *|*fd\ *|*ack\ *|*ag\ *) " - " [ -f graphify-out/graph.json ] && " - r""" echo '{"hookSpecificOutput":{"hookEventName":"PreToolUse","additionalContext":"MANDATORY: graphify-out/graph.json exists. You MUST run `graphify query \"\"` before grepping raw files. Only grep after graphify has oriented you, or to modify/debug specific lines."}}' """ - " || true ;; " - "esac" - ), - } - ], -} -_READ_SETTINGS_HOOK = { - # The Bash hook above never sees a file read through the native Read tool or a - # Glob, which is the most common way an agent skips the graph: answering a - # codebase question by Read-ing many source files one by one (issue #1114). - # Match Read|Glob, inspect the target path, and nudge (never block) only for a - # source/doc file outside graphify-out/ when a graph exists. The parser is - # python3 (already a graphify dependency), the shell is POSIX, and every branch - # fails open, so a legitimate read always goes through. Reading the graph's own - # report under graphify-out/ is suppressed so it never starts a feedback loop. - # The extension test compares each value's real trailing extension (segment - # after the last '/' then after the last '.') against exts -- not a substring - # scan, which both missed framework files like .astro and false-matched .json - # against .js (the substring '.js' is inside '.json'). - "matcher": "Read|Glob", - "hooks": [ - { - "type": "command", - "command": ( - "HIT=$(python3 -c \"" - "import json,sys;" - "d=json.load(sys.stdin);" - "t=d.get('tool_input',d);" - "exts=('.py','.js','.ts','.tsx','.jsx','.astro','.vue','.svelte','.go','.rs','.java','.rb','.c','.h','.cpp','.hpp','.cc','.cs','.kt','.swift','.php','.scala','.lua','.sh','.md','.rst','.txt','.mdx');" - "vals=[str(t.get('file_path') or ''),str(t.get('pattern') or ''),str(t.get('path') or '')];" - "j=' '.join(vals).lower().replace(chr(92),'/');" - "tails=[('.'+x.rsplit('.',1)[-1]) for v in vals if v for x in [v.lower().replace(chr(92),'/').rsplit('/',1)[-1]] if '.' in x];" - "sys.stdout.write('1' if 'graphify-out/' not in j and any(tl in exts for tl in tails) else '')\" 2>/dev/null || true); " - "if [ \"$HIT\" = 1 ] && [ -f graphify-out/graph.json ]; then " - r"""echo '{"hookSpecificOutput":{"hookEventName":"PreToolUse","additionalContext":"MANDATORY: graphify-out/graph.json exists. You MUST run graphify before reading source files. Use: `graphify query \"\"` (scoped subgraph), `graphify explain \"\"`, or `graphify path \"\" \"\"`. Only read raw files after graphify has oriented you, or to modify/debug specific lines. This rule applies to subagents too — include it in every subagent prompt involving code exploration."}}'; """ - "fi || true" - ), - } - ], -} -def _skill_registration(skill_path: str = "~/.claude/skills/graphify/SKILL.md") -> str: - return ( - "\n# graphify\n" - f"- **graphify** (`{skill_path}`) " - "- any input to knowledge graph. Trigger: `/graphify`\n" - "When the user types `/graphify`, use the installed graphify skill " - "or instructions before doing anything else.\n" - ) - - -_PLATFORM_CONFIG: dict[str, dict] = { - "claude": { - "skill_file": "skill.md", - "skill_dst": Path(".claude") / "skills" / "graphify" / "SKILL.md", - "claude_md": True, - "skill_refs": "claude", - }, - "codex": { - "skill_file": "skill-codex.md", - "skill_dst": Path(".codex") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "codex", - }, - "opencode": { - "skill_file": "skill-opencode.md", - "skill_dst": Path(".config") / "opencode" / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "opencode", - }, - "kilo": { - "skill_file": "skill-kilo.md", - "skill_dst": Path(".config") / "kilo" / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "kilo", - }, - "aider": { - # Monolith: aider ships the full SKILL.md inline, no references/ sidecar. - "skill_file": "skill-aider.md", - "skill_dst": Path(".aider") / "graphify" / "SKILL.md", - "claude_md": False, - }, - "copilot": { - "skill_file": "skill-copilot.md", - "skill_dst": Path(".copilot") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "copilot", - }, - "claw": { - "skill_file": "skill-claw.md", - "skill_dst": Path(".openclaw") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "claw", - }, - "droid": { - "skill_file": "skill-droid.md", - "skill_dst": Path(".factory") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "droid", - }, - "trae": { - "skill_file": "skill-trae.md", - "skill_dst": Path(".trae") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "trae", - }, - "trae-cn": { - # Reuses trae's split bundle (same skill body + references). - "skill_file": "skill-trae.md", - "skill_dst": Path(".trae-cn") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "trae", - }, - "hermes": { - # Reuses claw's split bundle. - "skill_file": "skill-claw.md", - "skill_dst": Path(".hermes") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "claw", - }, - "kiro": { - "skill_file": "skill-kiro.md", - "skill_dst": Path(".kiro") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "kiro", - }, - "pi": { - "skill_file": "skill-pi.md", - "skill_dst": Path(".pi") / "agent" / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "pi", - }, - "codebuddy": { - # Reuses claude's split bundle (shares skill.md). - "skill_file": "skill.md", - "skill_dst": Path(".codebuddy") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "claude", - }, - "antigravity": { - # Rides claude's split bundle (shares skill.md). - "skill_file": "skill.md", - "skill_dst": Path(".agents") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "claude", - }, - "antigravity-windows": { - # Rides windows' split bundle. - "skill_file": "skill-windows.md", - "skill_dst": Path(".agents") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "windows", - }, - "windows": { - "skill_file": "skill-windows.md", - "skill_dst": Path(".claude") / "skills" / "graphify" / "SKILL.md", - "claude_md": True, - "skill_refs": "windows", - }, - "kimi": { - # Reuses claude's split bundle (shares skill.md). - "skill_file": "skill.md", - "skill_dst": Path(".kimi") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "claude", - }, - "amp": { - # Amp searches .agents/skills (project) and ~/.config/agents/skills (user), - # not .amp/skills. The user-scope path is set in _platform_skill_destination. - "skill_file": "skill-amp.md", - "skill_dst": Path(".agents") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "amp", - }, - "agents": { - # The generic cross-framework Agent-Skills target. Global: ~/.agents/skills - # (the spec's user-global location, read by `npx skills` and compliant - # frameworks); project: ./.agents/skills. The CLI accepts `skills` as an - # alias (see _canonical_platform). Ships its own rendered bundle. - "skill_file": "skill-agents.md", - "skill_dst": Path(".agents") / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - "skill_refs": "agents", - }, - "devin": { - # Monolith: devin ships the full SKILL.md inline, no references/ sidecar. - "skill_file": "skill-devin.md", - # User scope: ~/.config/devin/skills/graphify/SKILL.md - # Project scope: .devin/skills/graphify/SKILL.md (overridden in _platform_skill_destination) - "skill_dst": Path(".config") / "devin" / "skills" / "graphify" / "SKILL.md", - "claude_md": False, - }, -} -# CLI-only platform aliases, resolved to a real _PLATFORM_CONFIG key before -# dispatch. `skills` is the friendly alias for the generic `agents` platform -# (the Agent-Skills ecosystem calls them "skills"). -_PLATFORM_ALIASES: dict[str, str] = {"skills": "agents"} -def _canonical_platform(platform_name: str) -> str: - """Resolve a CLI platform alias to its real _PLATFORM_CONFIG key.""" - return _PLATFORM_ALIASES.get(platform_name, platform_name) -def _replace_or_append_section(content: str, marker: str, new_section: str) -> str: - """Idempotently update or append a graphify-owned section in shared files. - If ``marker`` is not in ``content``, append ``new_section`` to the end - (with a blank-line separator if there's existing content). - If ``marker`` IS in ``content``, replace the existing section in place. - The section runs from the first line containing ``marker`` to the line - before the next H2 heading (``## `` at line start), or to EOF if no later - H2 exists. This lets older installs receive the updated copy without - users having to uninstall and reinstall — important for the issue #580 - fix where existing report-first text would otherwise silently linger. - """ - if marker not in content: - if content.strip(): - return content.rstrip() + "\n\n" + new_section.lstrip() - return new_section.lstrip() - - lines = content.split("\n") - start = next((i for i, line in enumerate(lines) if marker in line), None) - if start is None: - return content.rstrip() + "\n\n" + new_section.lstrip() - - end = len(lines) - for j in range(start + 1, len(lines)): - if lines[j].startswith("## "): - end = j - break - - head = "\n".join(lines[:start]).rstrip() - tail = "\n".join(lines[end:]).lstrip() - section = new_section.strip() - - parts: list[str] = [] - if head: - parts.append(head) - parts.append(section) - if tail: - parts.append(tail) - out = "\n\n".join(parts) - if not out.endswith("\n"): - out += "\n" - return out - - -def _print_banner() -> None: - """Amber brain banner on graphify install. TTY-only, never raises.""" - if not sys.stdout.isatty(): - return - try: - if sys.platform == "win32": - import ctypes - ctypes.windll.kernel32.SetConsoleMode( - ctypes.windll.kernel32.GetStdHandle(-11), 7 - ) - A = "\033[38;5;214m" - D = "\033[38;5;130m" - R = "\033[0m" - print(f"""{A} - ╭──◉──╮ ╭──◉──╮ - ╱ ◉ ◉ ╲ ╱ ◉ ◉ ╲ -│ ◉─◉─◉ ◉ ◉─◉─◉ │ -│ ◉ ◉ │ ◉ ◉ │ -│ ◉─◉─◉ ◉ ◉─◉─◉ │ - ╲ ◉ ◉ ╱ ╲ ◉ ◉ ╱ - ╰──◉──╯ ╰──◉──╯ - ◉ - - █▀▀ █▀█ ▄▀█ █▀█ █ █ █ █▀▀ █▄█ - █▄█ █▀▄ █▀█ █▀▀ █▀█ █ █▀ █{D} {__version__}{R} -""") - except Exception: - pass -def install(platform: str = "claude", *, project: bool = False, project_dir: Path | None = None) -> None: - _print_banner() - platform = _canonical_platform(platform) - if platform == "gemini": - gemini_install(project_dir=project_dir, project=project) - return - if platform == "cursor": - _cursor_install(Path(".")) - return - # On Windows, antigravity needs the PowerShell skill, not the bash one - if platform == "antigravity" and sys.platform == "win32": - platform = "antigravity-windows" - if platform not in _PLATFORM_CONFIG: - print( - f"error: unknown platform '{platform}'. Choose from: {', '.join(_PLATFORM_CONFIG)}, gemini, cursor", - file=sys.stderr, - ) - sys.exit(1) +# PreToolUse nudge payloads, emitted verbatim by the shell-agnostic +# `graphify hook-guard` subcommand (see _run_hook_guard). The previous hooks +# inlined POSIX bash (case/esac, [ -f ], single-quoted echo) which Windows +# cmd.exe/PowerShell cannot parse, so on Windows the hook failed and the nudge +# silently vanished — users had to invoke /graphify by hand (#522). Moving the +# logic into a Python subcommand invoked via an absolute exe path makes the hook +# parse identically under sh, cmd.exe and PowerShell. Claude Code accepts +# additionalContext on PreToolUse (Codex Desktop does not — that path stays a +# no-op via `hook-check`). Compact separators keep the payload byte-for-byte the +# same JSON the old `echo` emitted. + + +# Source/doc extensions the Read|Glob guard nudges on (verbatim from the old hook). +# The trailing-extension test (real final path segment, then its last '.') means +# '.json' never false-matches '.js', and framework files like '.astro' are kept. + + + + + + + + + - cfg = _PLATFORM_CONFIG[platform] - project_dir = project_dir or Path(".") - skill_dst = _copy_skill_file(platform, project=project, project_dir=project_dir) - if platform == "kilo": - # Kilo Code also supports a native /graphify command file. - command_src = Path(__file__).parent / "command-kilo.md" - if not command_src.exists(): - print( - f"error: command-kilo.md not found in package - reinstall graphify", - file=sys.stderr, - ) - sys.exit(1) - command_dst = Path.home() / ".config" / "kilo" / "command" / "graphify.md" - command_dst.parent.mkdir(parents=True, exist_ok=True) - shutil.copy(command_src, command_dst) - print(f" command installed -> {command_dst}") - - if cfg["claude_md"]: - # Register in the matching Claude Code scope. - claude_md = (project_dir / ".claude" / "CLAUDE.md") if project else Path.home() / ".claude" / "CLAUDE.md" - registration = _skill_registration(".claude/skills/graphify/SKILL.md" if project else "~/.claude/skills/graphify/SKILL.md") - if claude_md.exists(): - content = claude_md.read_text(encoding="utf-8") - if "graphify" in content: - print(f" CLAUDE.md -> already registered (no change)") - else: - claude_md.write_text(content.rstrip() + registration, encoding="utf-8") - print(f" CLAUDE.md -> skill registered in {claude_md}") - else: - claude_md.parent.mkdir(parents=True, exist_ok=True) - claude_md.write_text(registration.lstrip(), encoding="utf-8") - print(f" CLAUDE.md -> created at {claude_md}") - - if platform == "codebuddy": - # Register in ~/.codebuddy/CODEBUDDY.md (CodeBuddy only) - codebuddy_md = Path.home() / ".codebuddy" / "CODEBUDDY.md" - registration = _skill_registration("~/.codebuddy/skills/graphify/SKILL.md") - if codebuddy_md.exists(): - content = codebuddy_md.read_text(encoding="utf-8") - if "graphify" in content: - print(f" CODEBUDDY.md -> already registered (no change)") - else: - codebuddy_md.write_text(content.rstrip() + registration, encoding="utf-8") - print(f" CODEBUDDY.md -> skill registered in {codebuddy_md}") - else: - codebuddy_md.parent.mkdir(parents=True, exist_ok=True) - codebuddy_md.write_text(registration.lstrip(), encoding="utf-8") - print(f" CODEBUDDY.md -> created at {codebuddy_md}") - if platform == "opencode": - _install_opencode_plugin(project_dir if project else Path(".")) - # Refresh version stamps in all other previously-installed skill dirs so - # stale-version warnings don't fire for platforms not explicitly re-installed. - if project: - _print_project_git_add_hint([_project_scope_root(skill_dst, project_dir)]) - else: - _refresh_all_version_stamps() - print() - print("Done. Open your AI coding assistant and type:") - print() - print(" /graphify .") - print() -def _print_install_usage() -> None: - platforms = ", ".join([*_PLATFORM_CONFIG, "gemini", "cursor"]) - print("Usage: graphify install [--project] [--platform P|P]") - print(f"Platforms: {platforms}") # The always-on instruction blocks are packaged markdown under graphify/always_on/, @@ -827,749 +282,89 @@ def _print_install_usage() -> None: # load keeps the install-string / issue-#580 contract byte-for-byte while letting # a human edit one fragment instead of a triple-quoted literal here. -_CLAUDE_MD_MARKER = "## graphify" -_CODEBUDDY_MD_MARKER = "## graphify" # AGENTS.md section for Codex, OpenCode, and OpenClaw. # All three platforms read AGENTS.md in the project root for persistent instructions. -_AGENTS_MD_MARKER = "## graphify" - - -_GEMINI_MD_MARKER = "## graphify" - -_GEMINI_HOOK = { - "matcher": "read_file|list_directory", - "hooks": [ - { - "type": "command", - "command": ( - 'python -c "' - "import sys,pathlib,json;" - "e=pathlib.Path('graphify-out/graph.json').exists();" - "d={'decision':'allow'};" - "e and d.update({'additionalContext':'graphify: knowledge graph at graphify-out/. For focused questions, run `graphify query \"\"` (scoped subgraph, usually much smaller than GRAPH_REPORT.md) instead of grepping raw files. Read GRAPH_REPORT.md only for broad architecture context.'});" - "sys.stdout.write(json.dumps(d))" - '"' - ), - } - ], -} -def gemini_install(project_dir: Path | None = None, *, project: bool = False) -> None: - """Copy skill file, write GEMINI.md section, and install BeforeTool hook.""" - project_dir = project_dir or Path(".") - skill_dst = _copy_skill_file("gemini", project=project, project_dir=project_dir) - target = project_dir / "GEMINI.md" +# Gemini CLI BeforeTool hook nudge text. The hook always returns +# {"decision":"allow"} (never blocks a tool) and appends this as additionalContext +# when a graph exists. Emitted by `graphify hook-guard gemini`. The old hook was a +# `python -c "..."` one-liner that depended on a bare `python` on PATH (often +# `python`/`py` or absent on Windows) and embedded backticks + escaped quotes that +# Windows PowerShell mangles (#522 follow-up); the subcommand form has no such +# dependency and parses under every shell. + + + + + + + + + - if target.exists(): - content = target.read_text(encoding="utf-8") - new_content = _replace_or_append_section( - content, _GEMINI_MD_MARKER, _always_on("gemini-md") - ) - else: - new_content = _always_on("gemini-md") - - if target.exists() and new_content == target.read_text(encoding="utf-8"): - print(f"graphify already configured in {target.resolve()} (no change)") - else: - target.write_text(new_content, encoding="utf-8") - print(f"graphify section written to {target.resolve()}") - - # Always re-install the Gemini hook so an older payload (e.g. pre-issue-#580 - # wording) is replaced on upgrade. - _install_gemini_hook(project_dir) - if project: - _print_project_git_add_hint([_project_scope_root(skill_dst, project_dir), project_dir / "GEMINI.md", project_dir / ".gemini"]) - print() - print("Gemini CLI will now check the knowledge graph before answering") - print("codebase questions and rebuild it after code changes.") - - -def _install_gemini_hook(project_dir: Path) -> None: - settings_path = project_dir / ".gemini" / "settings.json" - settings_path.parent.mkdir(parents=True, exist_ok=True) - try: - settings = ( - json.loads(settings_path.read_text(encoding="utf-8")) - if settings_path.exists() - else {} - ) - except json.JSONDecodeError: - settings = {} - before_tool = settings.setdefault("hooks", {}).setdefault("BeforeTool", []) - settings["hooks"]["BeforeTool"] = [ - h for h in before_tool if "graphify" not in str(h) - ] - settings["hooks"]["BeforeTool"].append(_GEMINI_HOOK) - settings_path.write_text(json.dumps(settings, indent=2), encoding="utf-8") - print(" .gemini/settings.json -> BeforeTool hook registered") - - -def _uninstall_gemini_hook(project_dir: Path) -> None: - settings_path = project_dir / ".gemini" / "settings.json" - if not settings_path.exists(): - return - try: - settings = json.loads(settings_path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return - before_tool = settings.get("hooks", {}).get("BeforeTool", []) - filtered = [h for h in before_tool if "graphify" not in str(h)] - if len(filtered) == len(before_tool): - return - settings["hooks"]["BeforeTool"] = filtered - settings_path.write_text(json.dumps(settings, indent=2), encoding="utf-8") - print(" .gemini/settings.json -> BeforeTool hook removed") -def gemini_uninstall(project_dir: Path | None = None, *, project: bool = False) -> None: - """Remove the graphify section from GEMINI.md, uninstall hook, and remove skill file.""" - project_dir = project_dir or Path(".") - _remove_skill_file("gemini", project=project, project_dir=project_dir) - target = project_dir / "GEMINI.md" - if not target.exists(): - print("No GEMINI.md found in current directory - nothing to do") - return - content = target.read_text(encoding="utf-8") - if _GEMINI_MD_MARKER not in content: - print("graphify section not found in GEMINI.md - nothing to do") - return - cleaned = re.sub( - r"\n*## graphify\n.*?(?=\n## |\Z)", "", content, flags=re.DOTALL - ).rstrip() - if cleaned: - target.write_text(cleaned + "\n", encoding="utf-8") - print(f"graphify section removed from {target.resolve()}") - else: - target.unlink() - print(f"GEMINI.md was empty after removal - deleted {target.resolve()}") - _uninstall_gemini_hook(project_dir) - - -_VSCODE_INSTRUCTIONS_MARKER = "## graphify" - - -def vscode_install(project_dir: Path | None = None) -> None: - """Install graphify skill for VS Code Copilot Chat + write .github/copilot-instructions.md.""" - skill_src = Path(__file__).parent / "skill-vscode.md" - refs_bundle = "vscode" - if not skill_src.exists(): - skill_src = Path(__file__).parent / "skill-copilot.md" - refs_bundle = "copilot" - skill_dst = Path.home() / ".copilot" / "skills" / "graphify" / "SKILL.md" - skill_dst.parent.mkdir(parents=True, exist_ok=True) - tmp_dst = skill_dst.with_suffix(skill_dst.suffix + ".tmp") - try: - shutil.copy(skill_src, tmp_dst) - os.replace(tmp_dst, skill_dst) - except Exception: - try: - tmp_dst.unlink(missing_ok=True) - except OSError: - pass - raise - # Progressive-capable: install the packaged references/ sidecar when present. - refs_src = Path(__file__).parent / "skills" / refs_bundle / "references" - if refs_src.exists(): - _install_skill_references(skill_dst, refs_src) - print(f" references -> {skill_dst.parent / 'references'}") - else: - orphan_refs = skill_dst.parent / "references" - if orphan_refs.exists(): - shutil.rmtree(orphan_refs) - (skill_dst.parent / ".graphify_version").write_text(__version__, encoding="utf-8") - print(f" skill installed -> {skill_dst}") - - instructions = (project_dir or Path(".")) / ".github" / "copilot-instructions.md" - instructions.parent.mkdir(parents=True, exist_ok=True) - if instructions.exists(): - content = instructions.read_text(encoding="utf-8") - new_content = _replace_or_append_section( - content, _VSCODE_INSTRUCTIONS_MARKER, _always_on("vscode-instructions") - ) - if new_content == content: - print(f" {instructions} -> already configured (no change)") - else: - instructions.write_text(new_content, encoding="utf-8") - print(f" {instructions} -> graphify section {'updated' if _VSCODE_INSTRUCTIONS_MARKER in content else 'added'}") - else: - instructions.write_text(_always_on("vscode-instructions"), encoding="utf-8") - print(f" {instructions} -> created") - - print() - print( - "VS Code Copilot Chat configured. Type /graphify in the chat panel to build the graph." - ) - print("Note: for GitHub Copilot CLI (terminal), use: graphify copilot install") - - -def vscode_uninstall(project_dir: Path | None = None) -> None: - """Remove graphify VS Code Copilot Chat skill and .github/copilot-instructions.md section.""" - skill_dst = Path.home() / ".copilot" / "skills" / "graphify" / "SKILL.md" - if skill_dst.exists(): - skill_dst.unlink() - print(f" skill removed -> {skill_dst}") - version_file = skill_dst.parent / ".graphify_version" - if version_file.exists(): - version_file.unlink() - refs_dir = skill_dst.parent / "references" - if refs_dir.exists(): - shutil.rmtree(refs_dir) - for d in ( - skill_dst.parent, - skill_dst.parent.parent, - skill_dst.parent.parent.parent, - ): - try: - d.rmdir() - except OSError: - break - - instructions = (project_dir or Path(".")) / ".github" / "copilot-instructions.md" - if not instructions.exists(): - return - content = instructions.read_text(encoding="utf-8") - if _VSCODE_INSTRUCTIONS_MARKER not in content: - return - cleaned = re.sub( - r"\n*## graphify\n.*?(?=\n## |\Z)", "", content, flags=re.DOTALL - ).rstrip() - if cleaned: - instructions.write_text(cleaned + "\n", encoding="utf-8") - print(f" graphify section removed from {instructions}") - else: - instructions.unlink() - print(f" {instructions} -> deleted (was empty after removal)") -_ANTIGRAVITY_RULES_PATH = Path(".agents") / "rules" / "graphify.md" -_ANTIGRAVITY_WORKFLOW_PATH = Path(".agents") / "workflows" / "graphify.md" -_ANTIGRAVITY_WORKFLOW = """\ ---- -name: graphify -description: Turn any folder of files into a navigable knowledge graph ---- -# Workflow: graphify -Follow the graphify skill installed at ~/.gemini/config/skills/graphify/SKILL.md to run the full pipeline. -If no path argument is given, use `.` (current directory). -""" _KIRO_STEERING_MARKER = "graphify: A knowledge graph of this project" -def _kiro_install(project_dir: Path) -> None: - """Write graphify skill + steering file for Kiro IDE/CLI.""" - project_dir = project_dir or Path(".") - - # Skill file + references/ sidecar + .graphify_version stamp via the shared - # progressive-disclosure helper. Previously this used a bare write_text that - # bypassed _copy_skill_file, so the references/ dir and version stamp were - # never written even though kiro declares skill_refs: "kiro" (#1142). - _copy_skill_file("kiro", project=True, project_dir=project_dir) - - # Steering file → .kiro/steering/graphify.md (always-on) - steering_dir = project_dir / ".kiro" / "steering" - steering_dir.mkdir(parents=True, exist_ok=True) - steering_dst = steering_dir / "graphify.md" - if steering_dst.exists() and steering_dst.read_text(encoding="utf-8") == _always_on("kiro-steering"): - print(f" .kiro/steering/graphify.md -> already configured (no change)") - else: - # File is wholly graphify-owned. Overwrite on upgrade so older - # report-first wording does not silently linger (issue #580). - action = "updated" if steering_dst.exists() else "written" - steering_dst.write_text(_always_on("kiro-steering"), encoding="utf-8") - print(f" .kiro/steering/graphify.md -> always-on steering {action}") - - print() - print("Kiro will now read the knowledge graph before every conversation.") - print("Use /graphify to build or update the graph.") - - -def _kiro_uninstall(project_dir: Path) -> None: - """Remove graphify skill + steering file for Kiro.""" - project_dir = project_dir or Path(".") - removed = [] - - # Skill + .graphify_version + references/ sidecar + empty-dir walk. - skill_dst = _platform_skill_destination("kiro", project=True, project_dir=project_dir) - if _remove_skill_file("kiro", project=True, project_dir=project_dir): - removed.append(str(skill_dst.relative_to(project_dir))) - - steering_dst = project_dir / ".kiro" / "steering" / "graphify.md" - if steering_dst.exists(): - steering_dst.unlink() - removed.append(str(steering_dst.relative_to(project_dir))) - - print("Removed: " + (", ".join(removed) if removed else "nothing to remove")) - - -def _antigravity_finalize(skill_dst: Path, project_dir: Path) -> None: - """Write Antigravity's always-on layer next to an installed skill. - - Injects the native tool-discovery YAML frontmatter into *skill_dst*, then - writes ``.agents/rules/graphify.md`` and ``.agents/workflows/graphify.md`` - under *project_dir*. Shared by the global ``antigravity install`` and the - project-scoped ``install --project --platform antigravity`` paths, so both lay - down the rules/workflows that the uninstall path already expects to remove. - """ - # Inject YAML frontmatter for native Antigravity tool discovery. - if skill_dst.exists(): - content = skill_dst.read_text(encoding="utf-8") - if not content.startswith("---\n"): - frontmatter = "---\nname: graphify-manager\ndescription: Rebuild the code graph or perform manual CLI queries when MCP server is offline.\n---\n\n" - skill_dst.write_text(frontmatter + content, encoding="utf-8") - - # .agents/rules/graphify.md - rules_path = project_dir / _ANTIGRAVITY_RULES_PATH - rules_path.parent.mkdir(parents=True, exist_ok=True) - if rules_path.exists(): - existing = rules_path.read_text(encoding="utf-8") - if _always_on("antigravity-rules").strip() != existing.strip(): - rules_path.write_text(_always_on("antigravity-rules"), encoding="utf-8") - print(f"graphify rule updated at {rules_path.resolve()}") - else: - print(f"graphify rule already configured at {rules_path.resolve()} (no change)") - else: - rules_path.write_text(_always_on("antigravity-rules"), encoding="utf-8") - print(f"graphify rule written to {rules_path.resolve()}") - - # .agents/workflows/graphify.md - wf_path = project_dir / _ANTIGRAVITY_WORKFLOW_PATH - wf_path.parent.mkdir(parents=True, exist_ok=True) - if wf_path.exists(): - existing = wf_path.read_text(encoding="utf-8") - if _ANTIGRAVITY_WORKFLOW.strip() != existing.strip(): - wf_path.write_text(_ANTIGRAVITY_WORKFLOW, encoding="utf-8") - print(f"graphify workflow updated at {wf_path.resolve()}") - else: - print(f"graphify workflow already configured at {wf_path.resolve()} (no change)") - else: - wf_path.write_text(_ANTIGRAVITY_WORKFLOW, encoding="utf-8") - print(f"graphify workflow written to {wf_path.resolve()}") - - -def _antigravity_install(project_dir: Path) -> None: - """Install graphify for Google Antigravity (global skill + .agents/rules + .agents/workflows).""" - # Copy the skill to ~/.gemini/config/skills/graphify/SKILL.md (global), then - # lay down the always-on rules/workflows under the project dir. - install(platform="antigravity") - _antigravity_finalize(_platform_skill_destination("antigravity"), project_dir) - - print() - print("Antigravity will now check the knowledge graph before answering") - print("codebase questions. Run /graphify first to build the graph.") - print() - print( - "To enable full MCP architecture navigation, add this to ~/.gemini/antigravity/mcp_config.json:" - ) - print(' "graphify": {') - print(' "command": "uv",') - print( - ' "args": ["run", "--with", "graphifyy", "--with", "mcp", "-m", "graphify.serve", "${workspace.path}/graphify-out/graph.json"]' - ) - print(" }") - - -def _antigravity_uninstall(project_dir: Path, *, project: bool = False) -> None: - """Remove graphify Antigravity rules, workflow, and skill files.""" - # Remove rules file - rules_path = project_dir / _ANTIGRAVITY_RULES_PATH - if rules_path.exists(): - rules_path.unlink() - print(f"graphify rule removed from {rules_path.resolve()}") - else: - print("No graphify Antigravity rule found - nothing to do") - - # Remove workflow file - wf_path = project_dir / _ANTIGRAVITY_WORKFLOW_PATH - if wf_path.exists(): - wf_path.unlink() - print(f"graphify workflow removed from {wf_path.resolve()}") - - # Remove skill file - skill_dst = _platform_skill_destination("antigravity", project=project, project_dir=project_dir) - if skill_dst.exists(): - skill_dst.unlink() - print(f"graphify skill removed from {skill_dst}") - version_file = skill_dst.parent / ".graphify_version" - if version_file.exists(): - version_file.unlink() - refs_dir = skill_dst.parent / "references" - if refs_dir.exists(): - shutil.rmtree(refs_dir) - for d in ( - skill_dst.parent, - skill_dst.parent.parent, - skill_dst.parent.parent.parent, - ): - try: - d.rmdir() - except OSError: - break - - -_CURSOR_RULE_PATH = Path(".cursor") / "rules" / "graphify.mdc" -_CURSOR_RULE = """\ ---- -description: graphify knowledge graph context -alwaysApply: true ---- - -This project has a graphify knowledge graph at graphify-out/. - -**MANDATORY: Before using Read, Grep, Glob, or Bash to explore the codebase, you MUST run graphify first:** -- `graphify query ""` — scoped subgraph for any codebase or architecture question -- `graphify path "" ""` — dependency path between two symbols -- `graphify explain ""` — all nodes related to a concept - -This applies to YOU and to every subagent you spawn. Include this rule explicitly in every subagent prompt that involves code exploration. Do not skip graphify because files are "already known" or because you are executing a plan — the graph surfaces cross-file dependencies and INFERRED edges that grep and Read cannot find. - -Only use Read/Grep/Glob directly when: -1. graphify has already oriented you and you need to modify or debug specific lines -2. `graphify-out/graph.json` does not exist yet - -- If `graphify-out/wiki/index.md` exists, navigate it instead of reading raw files -- Read `graphify-out/GRAPH_REPORT.md` only for broad architecture review when query/path/explain do not surface enough context -- After modifying code files, run `graphify update .` to keep the graph current (AST-only, no API cost) -""" - - -def _cursor_install(project_dir: Path) -> None: - """Write .cursor/rules/graphify.mdc with alwaysApply: true.""" - rule_path = (project_dir or Path(".")) / _CURSOR_RULE_PATH - rule_path.parent.mkdir(parents=True, exist_ok=True) - if rule_path.exists() and rule_path.read_text(encoding="utf-8") == _CURSOR_RULE: - print(f"graphify rule at {rule_path} already configured (no change)") - return - # File is wholly graphify-owned. Overwrite on upgrade so older - # report-first wording does not silently linger (issue #580). - action = "updated" if rule_path.exists() else "written" - rule_path.write_text(_CURSOR_RULE, encoding="utf-8") - print(f"graphify rule {action} at {rule_path.resolve()}") - print() - print("Cursor will now always include the knowledge graph context.") - print("Run /graphify . first to build the graph if you haven't already.") - - -def _cursor_uninstall(project_dir: Path) -> None: - """Remove .cursor/rules/graphify.mdc.""" - rule_path = (project_dir or Path(".")) / _CURSOR_RULE_PATH - if not rule_path.exists(): - print("No graphify Cursor rule found - nothing to do") - return - rule_path.unlink() - print(f"graphify Cursor rule removed from {rule_path.resolve()}") -# Devin CLI — .windsurf/rules/graphify.md (always-on context) -# Devin reads .windsurf/rules/*.md files the same way Windsurf IDE does. -_DEVIN_RULES_PATH = Path(".windsurf") / "rules" / "graphify.md" -_DEVIN_RULES = """\ -## graphify -This project has a graphify knowledge graph at graphify-out/. -Rules: -- For codebase or architecture questions, when `graphify-out/graph.json` exists, first run `graphify query ""` (or `graphify path "" ""` / `graphify explain ""`). These return a scoped subgraph, usually much smaller than `GRAPH_REPORT.md` or raw grep output. -- If graphify-out/wiki/index.md exists, navigate it instead of reading raw files -- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context -- After modifying code files in this session, run `graphify update .` to keep the graph current (AST-only, no API cost) -""" -def _devin_rules_install(project_dir: Path) -> None: - """Write .windsurf/rules/graphify.md for always-on Devin context.""" - rules_path = (project_dir or Path(".")) / _DEVIN_RULES_PATH - rules_path.parent.mkdir(parents=True, exist_ok=True) - if rules_path.exists() and rules_path.read_text(encoding="utf-8") == _DEVIN_RULES: - print(f" {rules_path} -> already configured (no change)") - return - action = "updated" if rules_path.exists() else "written" - rules_path.write_text(_DEVIN_RULES, encoding="utf-8") - print(f" rules {action} -> {rules_path}") -def _devin_rules_uninstall(project_dir: Path) -> None: - """Remove .windsurf/rules/graphify.md.""" - rules_path = (project_dir or Path(".")) / _DEVIN_RULES_PATH - if not rules_path.exists(): - return - rules_path.unlink() - print(f" rules removed -> {rules_path}") - - -_KILO_PLUGIN_JS = """\ -// graphify Kilo plugin -// Injects a knowledge graph reminder before bash tool calls when the graph exists. -import { existsSync } from "fs"; -import { join } from "path"; - -export const GraphifyPlugin = async ({ directory }) => { - let reminded = false; - - return { - "tool.execute.before": async (input, output) => { - if (reminded) return; - if (!existsSync(join(directory, "graphify-out", "graph.json"))) return; - - if (input.tool === "bash") { - // Separate with ';' not '&&' — Windows PowerShell 5.1 rejects '&&' as a - // statement separator ("not a valid statement separator"), which broke - // the first bash command in every OpenCode session on Windows (#1646). - // ';' works in PowerShell 5.1, Bash, and POSIX shells alike. - output.args.command = - 'echo "[graphify] Knowledge graph available. Read graphify-out/GRAPH_REPORT.md for god nodes and architecture context before searching files." ; ' + - output.args.command; - reminded = true; - } - }, - }; -}; -""" - -_KILO_PLUGIN_PATH = Path(".kilo") / "plugins" / "graphify.js" -_KILO_CONFIG_JSON_PATH = Path(".kilo") / "kilo.json" -_KILO_CONFIG_JSONC_PATH = Path(".kilo") / "kilo.jsonc" - - -def _strip_json_comments(raw: str) -> str: - """Remove JSONC-style comments while leaving string content intact.""" - result: list[str] = [] - in_string = False - escaped = False - line_comment = False - block_comment = False - i = 0 - - while i < len(raw): - ch = raw[i] - nxt = raw[i + 1] if i + 1 < len(raw) else "" - - if line_comment: - if ch == "\n": - line_comment = False - result.append(ch) - i += 1 - continue - - if block_comment: - if ch == "*" and nxt == "/": - block_comment = False - i += 2 - else: - i += 1 - continue - - if in_string: - result.append(ch) - if escaped: - escaped = False - elif ch == "\\": - escaped = True - elif ch == '"': - in_string = False - i += 1 - continue - - if ch == "/" and nxt == "/": - line_comment = True - i += 2 - continue - if ch == "/" and nxt == "*": - block_comment = True - i += 2 - continue - - result.append(ch) - if ch == '"': - in_string = True - i += 1 - - return re.sub(r",(\s*[}\]])", r"\1", "".join(result)) - - -def _load_json_like(config_file: Path) -> dict: - if not config_file.exists(): - return {} - try: - raw = config_file.read_text(encoding="utf-8") - if config_file.suffix == ".jsonc": - raw = _strip_json_comments(raw) - loaded = json.loads(raw) - except (OSError, json.JSONDecodeError): - return {} - return loaded if isinstance(loaded, dict) else {} - - -def _kilo_config_path(project_dir: Path) -> Path: - kilo_dir = (project_dir or Path(".")) / ".kilo" - json_path = kilo_dir / _KILO_CONFIG_JSON_PATH.name - if json_path.exists(): - return json_path - jsonc_path = kilo_dir / _KILO_CONFIG_JSONC_PATH.name - if jsonc_path.exists(): - return jsonc_path - return json_path - - -def _kilo_config_write_path(project_dir: Path) -> Path: - """Write automated Kilo edits to kilo.json so existing JSONC stays untouched.""" - kilo_dir = (project_dir or Path(".")) / ".kilo" - return kilo_dir / _KILO_CONFIG_JSON_PATH.name - - -def _install_kilo_plugin(project_dir: Path) -> None: - """Write graphify.js plugin and register it without rewriting user JSONC.""" - plugin_file = project_dir / _KILO_PLUGIN_PATH - plugin_file.parent.mkdir(parents=True, exist_ok=True) - plugin_file.write_text(_KILO_PLUGIN_JS, encoding="utf-8") - print(f" {_KILO_PLUGIN_PATH} -> tool.execute.before hook written") - - config_file = _kilo_config_path(project_dir) - write_config_file = _kilo_config_write_path(project_dir) - write_config_file.parent.mkdir(parents=True, exist_ok=True) - config = _load_json_like(config_file) - plugins = config.get("plugin") - if not isinstance(plugins, list): - plugins = [] - config["plugin"] = plugins - entry = plugin_file.resolve().as_uri() - if entry not in plugins: - plugins.append(entry) - write_config_file.write_text(json.dumps(config, indent=2), encoding="utf-8") - print(f" {write_config_file.relative_to(project_dir)} -> plugin registered") - else: - print( - f" {config_file.relative_to(project_dir)} -> plugin already registered (no change)" - ) -def _uninstall_kilo_plugin(project_dir: Path) -> None: - """Remove graphify.js plugin and deregister it without rewriting user JSONC.""" - plugin_file = project_dir / _KILO_PLUGIN_PATH - if plugin_file.exists(): - plugin_file.unlink() - print(f" {_KILO_PLUGIN_PATH} -> removed") - config_file = _kilo_config_path(project_dir) - if not config_file.exists(): - return - write_config_file = _kilo_config_write_path(project_dir) - config = _load_json_like(config_file) - plugins = config.get("plugin", []) - if not isinstance(plugins, list): - plugins = [] - entry = plugin_file.resolve().as_uri() - if entry in plugins: - config["plugin"] = [plugin for plugin in plugins if plugin != entry] - if not config["plugin"]: - config.pop("plugin") - write_config_file.parent.mkdir(parents=True, exist_ok=True) - write_config_file.write_text(json.dumps(config, indent=2), encoding="utf-8") - print( - f" {write_config_file.relative_to(project_dir)} -> plugin deregistered" - ) -# OpenCode tool.execute.before plugin — fires before every tool call. -# Injects a graph reminder into bash command output when graph.json exists. -_OPENCODE_PLUGIN_JS = """\ -// graphify OpenCode plugin -// Injects a knowledge graph reminder before bash tool calls when the graph exists. -// -// IMPORTANT: keep the reminder string free of backticks and $(...) constructs. -// The hook prepends `echo "" && ` to the user's bash command; -// backticks inside the double-quoted echo trigger bash command substitution, -// which both corrupts tool output and silently executes the very graphify -// command we are only suggesting. Plain words render fine in opencode's TUI. -import { existsSync } from "fs"; -import { join } from "path"; - -export const GraphifyPlugin = async ({ directory }) => { - let reminded = false; - - return { - "tool.execute.before": async (input, output) => { - if (reminded) return; - if (!existsSync(join(directory, "graphify-out", "graph.json"))) return; - - if (input.tool === "bash") { - // ';' not '&&' — Windows PowerShell 5.1 rejects '&&' as a statement - // separator, breaking the first bash command of the session (#1646). - output.args.command = - 'echo "[graphify] knowledge graph at graphify-out/. For focused questions, run graphify query with your question (scoped subgraph, usually much smaller than GRAPH_REPORT.md) instead of grepping raw files. Read GRAPH_REPORT.md only for broad architecture context." ; ' + - output.args.command; - reminded = true; - } - }, - }; -}; -""" - -_OPENCODE_PLUGIN_PATH = Path(".opencode") / "plugins" / "graphify.js" -_OPENCODE_CONFIG_PATH = Path(".opencode") / "opencode.json" - - -def _install_opencode_plugin(project_dir: Path) -> None: - """Write graphify.js plugin and register it in opencode.json.""" - plugin_file = project_dir / _OPENCODE_PLUGIN_PATH - plugin_file.parent.mkdir(parents=True, exist_ok=True) - plugin_file.write_text(_OPENCODE_PLUGIN_JS, encoding="utf-8") - print(f" {_OPENCODE_PLUGIN_PATH} -> tool.execute.before hook written") - - config_file = project_dir / _OPENCODE_CONFIG_PATH - if config_file.exists(): - try: - config = json.loads(config_file.read_text(encoding="utf-8")) - except json.JSONDecodeError: - config = {} - else: - config = {} - - plugins = config.setdefault("plugin", []) - entry = _OPENCODE_PLUGIN_PATH.as_posix() - if entry not in plugins: - plugins.append(entry) - config_file.write_text(json.dumps(config, indent=2), encoding="utf-8") - print(f" {_OPENCODE_CONFIG_PATH} -> plugin registered") - else: - print(f" {_OPENCODE_CONFIG_PATH} -> plugin already registered (no change)") - - -def _uninstall_opencode_plugin(project_dir: Path) -> None: - """Remove graphify.js plugin and deregister from opencode.json.""" - plugin_file = project_dir / _OPENCODE_PLUGIN_PATH - if plugin_file.exists(): - plugin_file.unlink() - print(f" {_OPENCODE_PLUGIN_PATH} -> removed") - - config_file = project_dir / _OPENCODE_CONFIG_PATH - if not config_file.exists(): - return - try: - config = json.loads(config_file.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return - plugins = config.get("plugin", []) - entry = _OPENCODE_PLUGIN_PATH.as_posix() - if entry in plugins: - plugins.remove(entry) - if not plugins: - config.pop("plugin") - config_file.write_text(json.dumps(config, indent=2), encoding="utf-8") - print(f" {_OPENCODE_CONFIG_PATH} -> plugin deregistered") + + + + + + + + + + + + + + + + + + + + + + + + + + + + + _CODEX_HOOK = { @@ -1593,648 +388,99 @@ def _uninstall_opencode_plugin(project_dir: Path) -> None: } -def _resolve_graphify_exe() -> str: - """Return the absolute path to the graphify executable. - Falls back to bare 'graphify' if resolution fails. Using an absolute path - ensures the hook works in environments where the venv Scripts/ directory is - not on PATH (e.g. VS Code Codex extension on Windows). - """ - import shutil - found = shutil.which("graphify") - if found: - return found - # Derive from sys.executable: same Scripts/ (Windows) or bin/ (Unix) dir - scripts_dir = Path(sys.executable).parent - for name in ("graphify.exe", "graphify"): - candidate = scripts_dir / name - if candidate.exists(): - return str(candidate) - return "graphify" - - -def _install_codex_hook(project_dir: Path) -> None: - """Add graphify PreToolUse hook to .codex/hooks.json.""" - hooks_path = project_dir / ".codex" / "hooks.json" - hooks_path.parent.mkdir(parents=True, exist_ok=True) - - if hooks_path.exists(): - try: - existing = json.loads(hooks_path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - existing = {} - else: - existing = {} - - graphify_exe = _resolve_graphify_exe() - hook_entry = { - "hooks": { - "PreToolUse": [ - { - "matcher": "Bash", - "hooks": [{"type": "command", "command": f"{graphify_exe} hook-check"}], - } - ] - } - } - pre_tool = existing.setdefault("hooks", {}).setdefault("PreToolUse", []) - existing["hooks"]["PreToolUse"] = [h for h in pre_tool if "graphify" not in str(h)] - existing["hooks"]["PreToolUse"].extend(hook_entry["hooks"]["PreToolUse"]) - hooks_path.write_text(json.dumps(existing, indent=2), encoding="utf-8") - print(f" .codex/hooks.json -> PreToolUse hook registered ({graphify_exe} hook-check)") -def _uninstall_codex_hook(project_dir: Path) -> None: - """Remove graphify PreToolUse hook from .codex/hooks.json.""" - hooks_path = project_dir / ".codex" / "hooks.json" - if not hooks_path.exists(): - return - try: - existing = json.loads(hooks_path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return - pre_tool = existing.get("hooks", {}).get("PreToolUse", []) - filtered = [h for h in pre_tool if "graphify" not in str(h)] - existing["hooks"]["PreToolUse"] = filtered - hooks_path.write_text(json.dumps(existing, indent=2), encoding="utf-8") - print(f" .codex/hooks.json -> PreToolUse hook removed") -def _agents_install(project_dir: Path, platform: str) -> None: - """Write the graphify section to the local AGENTS.md for always-on platforms.""" - target = (project_dir or Path(".")) / "AGENTS.md" - if target.exists(): - content = target.read_text(encoding="utf-8") - new_content = _replace_or_append_section( - content, _AGENTS_MD_MARKER, _always_on("agents-md") - ) - else: - new_content = _always_on("agents-md") - - if target.exists() and new_content == target.read_text(encoding="utf-8"): - print(f"graphify already configured in {target.resolve()} (no change)") - else: - target.write_text(new_content, encoding="utf-8") - print(f"graphify section written to {target.resolve()}") - - if platform == "codex": - _install_codex_hook(project_dir or Path(".")) - elif platform == "opencode": - _install_opencode_plugin(project_dir or Path(".")) - elif platform == "kilo": - _install_kilo_plugin(project_dir or Path(".")) - - print() - print( - f"{platform.capitalize()} will now check the knowledge graph before answering" - ) - print("codebase questions and rebuild it after code changes.") - if platform not in ("codex", "opencode", "kilo"): - print() - print("Note: unlike Claude Code, there is no PreToolUse hook equivalent for") - print( - f"{platform.capitalize()} — the AGENTS.md rules are the always-on mechanism." - ) -def _amp_legacy_cleanup() -> None: - """Best-effort removal of the pre-fix ~/.amp/skills/graphify install dir. - Older graphify versions wrote the Amp skill to ~/.amp/skills, which Amp does - not search. Clean it up on install so a stale, never-loaded copy does not - linger. Failures are ignored (the new path is what matters). - """ - legacy = Path.home() / ".amp" / "skills" / "graphify" - if legacy.exists(): - shutil.rmtree(legacy, ignore_errors=True) - if not legacy.exists(): - print(f" legacy removed -> {legacy}") -def _amp_install(project_dir: Path | None = None) -> None: - """User-scope Amp install: skill into ~/.config/agents/skills + AGENTS.md.""" - _amp_legacy_cleanup() - _copy_skill_file("amp") - _agents_install(project_dir or Path("."), "amp") -def _amp_uninstall(project_dir: Path | None = None) -> None: - """User-scope Amp uninstall: remove the skill and the AGENTS.md section.""" - removed = _remove_skill_file("amp") - if removed: - print("skill removed") - _agents_uninstall(project_dir or Path("."), platform="amp") -def _agents_platform_install(project_dir: Path | None = None) -> None: - """`graphify agents install`: skill into ~/.agents/skills + AGENTS.md. - The amp-twin of the generic Agent-Skills target. Mirrors _amp_install but - lands the skill at the spec's user-global ~/.agents/skills (set in - _platform_skill_destination). Wiring AGENTS.md keeps it honest with the - rendered hooks reference, which points at `graphify agents install`. The bare - `graphify install --platform agents` path stays skill-only (via install()), - exactly as amp's `--platform amp` does. - """ - _copy_skill_file("agents") - _agents_install(project_dir or Path("."), "agents") - - -def _agents_platform_uninstall(project_dir: Path | None = None) -> None: - """`graphify agents uninstall`: remove the skill and the AGENTS.md section.""" - removed = _remove_skill_file("agents") - if removed: - print("skill removed") - _agents_uninstall(project_dir or Path("."), platform="agents") - - -def _project_install(platform_name: str, project_dir: Path | None = None) -> None: - """Install platform skill/config files in the current project.""" - project_dir = project_dir or Path(".") - platform_name = _canonical_platform(platform_name) - if platform_name in ("claude", "windows"): - install(platform=platform_name, project=True, project_dir=project_dir) - claude_install(project_dir) - _print_project_git_add_hint([project_dir / ".claude", project_dir / "CLAUDE.md"]) - elif platform_name == "gemini": - gemini_install(project_dir, project=True) - elif platform_name == "cursor": - _cursor_install(project_dir) - _print_project_git_add_hint([project_dir / ".cursor"]) - elif platform_name == "kiro": - _kiro_install(project_dir) - _print_project_git_add_hint([project_dir / ".kiro"]) - elif platform_name in ("aider", "amp", "codex", "opencode", "claw", "droid", "trae", "trae-cn", "hermes"): - skill_dst = _copy_skill_file(platform_name, project=True, project_dir=project_dir) - _agents_install(project_dir, platform_name) - hint_paths = [_project_scope_root(skill_dst, project_dir), project_dir / "AGENTS.md"] - if platform_name == "opencode": - hint_paths.append(project_dir / ".opencode") - elif platform_name == "codex": - hint_paths.append(project_dir / ".codex") - _print_project_git_add_hint(hint_paths) - elif platform_name == "devin": - skill_dst = _copy_skill_file("devin", project=True, project_dir=project_dir) - _devin_rules_install(project_dir) - _print_project_git_add_hint([_project_scope_root(skill_dst, project_dir), project_dir / ".windsurf"]) - elif platform_name == "antigravity": - # Project-scoped: skill in .agents/skills/ PLUS the .agents/rules + - # .agents/workflows always-on layer (previously this path wrote only the - # skill, leaving the rules/workflows the uninstall path removes unset). - skill_dst = _copy_skill_file("antigravity", project=True, project_dir=project_dir) - _antigravity_finalize(skill_dst, project_dir) - _print_project_git_add_hint([_project_scope_root(skill_dst, project_dir), project_dir / ".agents"]) - elif platform_name in ("copilot", "pi", "kimi", "agents"): - # Skill-only project install: drop SKILL.md (+ references) at the scope - # root. `agents` -> ./.agents/skills/graphify/SKILL.md. - skill_dst = _copy_skill_file(platform_name, project=True, project_dir=project_dir) - _print_project_git_add_hint([_project_scope_root(skill_dst, project_dir)]) - else: - install(platform=platform_name, project=True, project_dir=project_dir) - - -def _project_uninstall(platform_name: str, project_dir: Path | None = None) -> None: - """Remove project-scoped platform skill/config files only.""" - project_dir = project_dir or Path(".") - platform_name = _canonical_platform(platform_name) - if platform_name in ("claude", "windows"): - _remove_skill_file(platform_name, project=True, project_dir=project_dir) - _remove_claude_skill_registration(project_dir) - claude_uninstall(project_dir, project=True) - elif platform_name == "gemini": - gemini_uninstall(project_dir, project=True) - elif platform_name == "cursor": - _cursor_uninstall(project_dir) - elif platform_name == "kiro": - _kiro_uninstall(project_dir) - elif platform_name in ("aider", "amp", "codex", "opencode", "claw", "droid", "trae", "trae-cn", "hermes"): - _remove_skill_file(platform_name, project=True, project_dir=project_dir) - _agents_uninstall(project_dir, platform=platform_name) - if platform_name == "codex": - _uninstall_codex_hook(project_dir) - elif platform_name == "antigravity": - _antigravity_uninstall(project_dir, project=True) - elif platform_name == "devin": - removed = _remove_skill_file("devin", project=True, project_dir=project_dir) - _devin_rules_uninstall(project_dir) - if not removed: - print("nothing to remove") - elif platform_name in ("copilot", "pi", "kimi", "agents"): - removed = _remove_skill_file(platform_name, project=True, project_dir=project_dir) - if not removed: - print("nothing to remove") - elif platform_name == "codebuddy": - codebuddy_uninstall(project_dir) - else: - _remove_skill_file(platform_name, project=True, project_dir=project_dir) - - -def _project_uninstall_all(project_dir: Path | None = None) -> None: - """Remove project-scoped install files without touching user-scope installs.""" - project_dir = project_dir or Path(".") - print("Uninstalling project-scoped graphify files...\n") - for platform_name in _PLATFORM_CONFIG: - _project_uninstall(platform_name, project_dir) - for platform_name in ("gemini", "cursor"): - _project_uninstall(platform_name, project_dir) - print("\nDone.") - - -def _agents_uninstall(project_dir: Path, platform: str = "") -> None: - """Remove the graphify section from the local AGENTS.md.""" - target = (project_dir or Path(".")) / "AGENTS.md" - - if not target.exists(): - print("No AGENTS.md found in current directory - nothing to do") - if platform == "opencode": - _uninstall_opencode_plugin(project_dir or Path(".")) - elif platform == "kilo": - _uninstall_kilo_plugin(project_dir or Path(".")) - return - content = target.read_text(encoding="utf-8") - if _AGENTS_MD_MARKER not in content: - print("graphify section not found in AGENTS.md - nothing to do") - if platform == "opencode": - _uninstall_opencode_plugin(project_dir or Path(".")) - elif platform == "kilo": - _uninstall_kilo_plugin(project_dir or Path(".")) - return - cleaned = re.sub( - r"\n*## graphify\n.*?(?=\n## |\Z)", - "", - content, - flags=re.DOTALL, - ).rstrip() - if cleaned: - target.write_text(cleaned + "\n", encoding="utf-8") - print(f"graphify section removed from {target.resolve()}") - else: - target.unlink() - print(f"AGENTS.md was empty after removal - deleted {target.resolve()}") - - if platform == "opencode": - _uninstall_opencode_plugin(project_dir or Path(".")) - elif platform == "kilo": - _uninstall_kilo_plugin(project_dir or Path(".")) - - -def _kilo_uninstall_global() -> list[str]: - removed = [] - command_dst = Path.home() / ".config" / "kilo" / "command" / "graphify.md" - if command_dst.exists(): - command_dst.unlink() - removed.append(f"command removed: {command_dst}") - try: - command_dst.parent.rmdir() - except OSError: - pass - skill_dst = Path.home() / _PLATFORM_CONFIG["kilo"]["skill_dst"] - if skill_dst.exists(): - skill_dst.unlink() - removed.append(f"skill removed: {skill_dst}") - version_file = skill_dst.parent / ".graphify_version" - if version_file.exists(): - version_file.unlink() - for d in ( - skill_dst.parent, - skill_dst.parent.parent, - skill_dst.parent.parent.parent, - ): - try: - d.rmdir() - except OSError: - break - - return removed - - -def _kilo_install(project_dir: Path) -> None: - """Install native Kilo skill + command globally and always-on project wiring locally.""" - install(platform="kilo") - _agents_install(project_dir or Path("."), "kilo") - - -def _kilo_uninstall(project_dir: Path) -> None: - """Remove Kilo always-on project wiring and global skill/command files.""" - _agents_uninstall(project_dir or Path("."), platform="kilo") - removed = _kilo_uninstall_global() - print("; ".join(removed) if removed else "nothing to remove") - - -def claude_install(project_dir: Path | None = None) -> None: - """Write the graphify section to the local CLAUDE.md.""" - target = (project_dir or Path(".")) / "CLAUDE.md" - - if target.exists(): - content = target.read_text(encoding="utf-8") - new_content = _replace_or_append_section( - content, _CLAUDE_MD_MARKER, _always_on("claude-md") - ) - else: - new_content = _always_on("claude-md") - - if target.exists() and new_content == target.read_text(encoding="utf-8"): - print(f"graphify already configured in {target.resolve()} (no change)") - else: - target.write_text(new_content, encoding="utf-8") - print(f"graphify section written to {target.resolve()}") - - # Always re-install the Claude Code PreToolUse hook so an old hook - # payload (e.g. pre-issue-#580 wording) is replaced on upgrade. - _install_claude_hook(project_dir or Path(".")) - - print() - print("Claude Code will now check the knowledge graph before answering") - print("codebase questions and rebuild it after code changes.") - - -def _install_claude_hook(project_dir: Path) -> None: - """Add graphify PreToolUse hook to .claude/settings.json.""" - settings_path = project_dir / ".claude" / "settings.json" - settings_path.parent.mkdir(parents=True, exist_ok=True) - - if settings_path.exists(): - try: - settings = json.loads(settings_path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - settings = {} - else: - settings = {} - - hooks = settings.setdefault("hooks", {}) - pre_tool = hooks.setdefault("PreToolUse", []) - - hooks["PreToolUse"] = [h for h in pre_tool if not (h.get("matcher") in ("Glob|Grep", "Bash", "Read|Glob") and "graphify" in str(h))] - hooks["PreToolUse"].append(_SETTINGS_HOOK) - hooks["PreToolUse"].append(_READ_SETTINGS_HOOK) - settings_path.write_text(json.dumps(settings, indent=2), encoding="utf-8") - print(f" .claude/settings.json -> PreToolUse hooks registered (Bash search + Read/Glob)") - - -def _uninstall_claude_hook(project_dir: Path) -> None: - """Remove graphify PreToolUse hook from .claude/settings.json.""" - settings_path = project_dir / ".claude" / "settings.json" - if not settings_path.exists(): - return - try: - settings = json.loads(settings_path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return - pre_tool = settings.get("hooks", {}).get("PreToolUse", []) - filtered = [h for h in pre_tool if not (h.get("matcher") in ("Glob|Grep", "Bash", "Read|Glob") and "graphify" in str(h))] - if len(filtered) == len(pre_tool): - return - settings["hooks"]["PreToolUse"] = filtered - settings_path.write_text(json.dumps(settings, indent=2), encoding="utf-8") - print(f" .claude/settings.json -> PreToolUse hook removed") - - -def uninstall_all(project_dir: Path | None = None, purge: bool = False) -> None: - """Remove graphify from every platform detected in the current project.""" - pd = project_dir or Path(".") - print("Uninstalling graphify from all detected platforms...\n") - - # Skill-file / config-section uninstallers - claude_uninstall(pd) - codebuddy_uninstall(pd) - gemini_uninstall(pd) - vscode_uninstall(pd) - _cursor_uninstall(pd) - _kiro_uninstall(pd) - _antigravity_uninstall(pd) - # AGENTS.md covers: codex, aider, opencode, claw, droid, trae, trae-cn, hermes, copilot - _agents_uninstall(pd) - # Amp also drops a user-scope skill at ~/.config/agents/skills, which the - # AGENTS.md cleanup above does not touch. - _remove_skill_file("amp") - # The generic agents platform's user-scope skill lives at ~/.agents/skills, - # which neither the AGENTS.md cleanup nor amp's removal reaches. - _remove_skill_file("agents") - _uninstall_opencode_plugin(pd) - _uninstall_codex_hook(pd) - - # Git hook - try: - from graphify.hooks import uninstall as hook_uninstall - result = hook_uninstall(pd) - if result: - print(result) - except Exception: - pass - if purge: - import shutil as _shutil - out = pd / _GRAPHIFY_OUT - if out.exists(): - _shutil.rmtree(out) - print(f"\n {_GRAPHIFY_OUT}/ -> deleted (--purge)") - else: - print(f"\n {_GRAPHIFY_OUT}/ -> not found (nothing to purge)") - print("\nDone. Run 'pip uninstall graphifyy' to remove the package itself.") -def claude_uninstall(project_dir: Path | None = None, *, project: bool = False) -> None: - """Remove the graphify skill tree (SKILL.md + references/) and the CLAUDE.md section. - Mirrors gemini_uninstall: the bare `graphify uninstall` and `graphify claude - uninstall` must remove the installed skill, not just strip CLAUDE.md, or the - progressive-disclosure tree (SKILL.md + references/) is orphaned (#1121). - """ - project_dir = project_dir or Path(".") - _remove_skill_file("claude", project=project, project_dir=project_dir) - target = project_dir / "CLAUDE.md" - if not target.exists(): - print("No CLAUDE.md found in current directory - nothing to do") - return - content = target.read_text(encoding="utf-8") - if _CLAUDE_MD_MARKER not in content: - print("graphify section not found in CLAUDE.md - nothing to do") - return - # Remove the ## graphify section: from the marker to the next ## heading or EOF - cleaned = re.sub( - r"\n*## graphify\n.*?(?=\n## |\Z)", - "", - content, - flags=re.DOTALL, - ).rstrip() - if cleaned: - target.write_text(cleaned + "\n", encoding="utf-8") - print(f"graphify section removed from {target.resolve()}") - else: - target.unlink() - print(f"CLAUDE.md was empty after removal - deleted {target.resolve()}") - - _uninstall_claude_hook(project_dir or Path(".")) - - -def codebuddy_install(project_dir: Path | None = None) -> None: - """Install the graphify skill and CODEBUDDY.md section for CodeBuddy.""" - _copy_skill_file("codebuddy", project=bool(project_dir), project_dir=project_dir) - target = (project_dir or Path(".")) / "CODEBUDDY.md" - - if target.exists(): - content = target.read_text(encoding="utf-8") - new_content = _replace_or_append_section( - content, _CODEBUDDY_MD_MARKER, _always_on("claude-md") - ) - else: - new_content = _always_on("claude-md") - - if target.exists() and new_content == target.read_text(encoding="utf-8"): - print(f"graphify already configured in {target.resolve()} (no change)") - else: - target.write_text(new_content, encoding="utf-8") - print(f"graphify section written to {target.resolve()}") - - # Also write CodeBuddy PreToolUse hook to .codebuddy/settings.json - _install_codebuddy_hook(project_dir or Path(".")) - - print() - print("CodeBuddy will now check the knowledge graph before answering") - print("codebase questions and rebuild it after code changes.") - - -def _install_codebuddy_hook(project_dir: Path) -> None: - """Add graphify PreToolUse hook to .codebuddy/settings.json.""" - settings_path = project_dir / ".codebuddy" / "settings.json" - settings_path.parent.mkdir(parents=True, exist_ok=True) - - if settings_path.exists(): - try: - settings = json.loads(settings_path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - settings = {} - else: - settings = {} - - hooks = settings.setdefault("hooks", {}) - pre_tool = hooks.setdefault("PreToolUse", []) - - hooks["PreToolUse"] = [h for h in pre_tool if not (h.get("matcher") in ("Glob|Grep", "Bash", "Read|Glob") and "graphify" in str(h))] - hooks["PreToolUse"].append(_SETTINGS_HOOK) - hooks["PreToolUse"].append(_READ_SETTINGS_HOOK) - settings_path.write_text(json.dumps(settings, indent=2), encoding="utf-8") - print(f" .codebuddy/settings.json -> PreToolUse hooks registered") - - -def _uninstall_codebuddy_hook(project_dir: Path) -> None: - """Remove graphify PreToolUse hook from .codebuddy/settings.json.""" - settings_path = project_dir / ".codebuddy" / "settings.json" - if not settings_path.exists(): - return - try: - settings = json.loads(settings_path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return - pre_tool = settings.get("hooks", {}).get("PreToolUse", []) - filtered = [h for h in pre_tool if not (h.get("matcher") in ("Glob|Grep", "Bash", "Read|Glob") and "graphify" in str(h))] - if len(filtered) == len(pre_tool): - return - settings["hooks"]["PreToolUse"] = filtered - settings_path.write_text(json.dumps(settings, indent=2), encoding="utf-8") - print(f" .codebuddy/settings.json -> PreToolUse hook removed") -def codebuddy_uninstall(project_dir: Path | None = None, *, project: bool = False) -> None: - """Remove the graphify skill tree (SKILL.md + references/) and the CODEBUDDY.md section.""" - project_dir = project_dir or Path(".") - _remove_skill_file("codebuddy", project=project, project_dir=project_dir) - target = project_dir / "CODEBUDDY.md" - if not target.exists(): - print("No CODEBUDDY.md found in current directory - nothing to do") - return - content = target.read_text(encoding="utf-8") - if _CODEBUDDY_MD_MARKER not in content: - print("graphify section not found in CODEBUDDY.md - nothing to do") - return - # Remove the ## graphify section: from the marker to the next ## heading or EOF - cleaned = re.sub( - r"\n*## graphify\n.*?(?=\n## |\Z)", - "", - content, - flags=re.DOTALL, - ).rstrip() - if cleaned: - target.write_text(cleaned + "\n", encoding="utf-8") - print(f"graphify section removed from {target.resolve()}") - else: - target.unlink() - print(f"CODEBUDDY.md was empty after removal - deleted {target.resolve()}") - - _uninstall_codebuddy_hook(project_dir or Path(".")) - -def _clone_repo( - url: str, branch: str | None = None, out_dir: Path | None = None -) -> Path: - """Clone a GitHub repo to a local cache dir and return the path. - - Clones into ~/.graphify/repos// by default so repeated - runs on the same URL reuse the existing clone (git pull instead of clone). - """ - import subprocess as _sp - import re as _re - - # Normalise URL — strip trailing .git if present - url = url.rstrip("/") - if not url.endswith(".git"): - git_url = url + ".git" - else: - git_url = url - url = url[:-4] - - # Extract owner/repo from URL - m = _re.search(r"github\.com[:/]([^/]+)/([^/]+?)(?:\.git)?$", url) - if not m: - print(f"error: not a recognised GitHub URL: {url}", file=sys.stderr) - sys.exit(1) - owner, repo = m.group(1), m.group(2) - - if out_dir: - dest = out_dir - else: - dest = Path.home() / ".graphify" / "repos" / owner / repo - - if branch and branch.startswith("-"): - print(f"error: invalid branch name: {branch!r}", file=sys.stderr) - sys.exit(1) - - if dest.exists(): - print(f"Repo already cloned at {dest} - pulling latest...", flush=True) - cmd = ["git", "-C", str(dest), "pull"] - if branch: - cmd += ["origin", "--", branch] - result = _sp.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - print(f"warning: git pull failed:\n{result.stderr}", file=sys.stderr) - else: - dest.parent.mkdir(parents=True, exist_ok=True) - print(f"Cloning {url} -> {dest} ...", flush=True) - cmd = ["git", "clone", "--depth", "1"] - if branch: - cmd += ["--branch", branch] - cmd += ["--", git_url, str(dest)] - result = _sp.run(cmd, capture_output=True, text=True) - if result.returncode != 0: - print(f"error: git clone failed:\n{result.stderr}", file=sys.stderr) - sys.exit(1) - - print(f"Ready at: {dest}", flush=True) - return dest + + + + + + + + + + + + + + + + + + + + + + + + +def _silence_broken_pipe() -> None: + """Handle a downstream reader that closed the pipe early. Redirect stdout to + devnull so the interpreter's shutdown flush does not raise a second time, then + exit 0 — the reader (head, `Select-Object -First N`, `sed q`) has what it needs.""" + try: + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, sys.stdout.fileno()) + except Exception: + pass + sys.exit(0) def main() -> None: + """Console entry point. Wraps the CLI so that when a downstream consumer closes + stdout early, graphify treats it as success instead of crashing with an + unhandled write-to-closed-pipe error and exit 255 — which made CI wrappers and + agent harnesses read a successful query as a command failure (#1807).""" + try: + _run_cli() + # Flush explicitly, inside the guard. Piped stdout is block-buffered, so a + # small fully-buffered output would otherwise only flush at interpreter + # shutdown — outside this try — where a reader that closed the pipe surfaces + # as a noisy "Exception ignored on flushing sys.stdout" and a nonzero exit. + sys.stdout.flush() + except BrokenPipeError: + _silence_broken_pipe() + except OSError as exc: + # Windows surfaces a write to a closed pipe as OSError(EINVAL) rather than + # BrokenPipeError; EPIPE is the POSIX form when it slips past the above. + if getattr(exc, "errno", None) in (errno.EPIPE, errno.EINVAL): + _silence_broken_pipe() + else: + raise + + +def _run_cli() -> None: for _stream in (sys.stdout, sys.stderr): if _stream is not None and hasattr(_stream, "reconfigure"): try: @@ -2245,7 +491,7 @@ def main() -> None: # Skip during install/uninstall (hook writes trigger a fresh check anyway). # Skip during hook-check — it runs on every editor tool use and must be silent. # Deduplicate paths so platforms sharing the same install dir don't warn twice. - _silent_cmds = {"install", "uninstall", "hook-check"} + _silent_cmds = {"install", "uninstall", "hook-check", "hook-guard"} if not any(arg in _silent_cmds for arg in sys.argv): # Resolve each platform's real user-scope destination so per-platform # overrides (gemini, opencode, devin, antigravity, amp) check the dir @@ -2316,6 +562,10 @@ def main() -> None: print(" --relation R edge relation to traverse in reverse (repeatable)") print(" --depth N reverse traversal depth (default 2)") print(" --graph path to graph.json (default graphify-out/graph.json)") + print(" god-nodes list the most connected nodes (architectural hubs)") + print(" --top N how many to show (default 10)") + print(" --graph path to graph.json (default graphify-out/graph.json)") + print(" --json emit JSON instead of text") print(" save-result save a Q&A result to graphify-out/memory/ for graph feedback loop") print(" --question Q the question asked") print(" --answer A the answer to save") @@ -2351,13 +601,17 @@ def main() -> None: print(" proxy, gateways): set ANTHROPIC_BASE_URL and ANTHROPIC_MODEL") print(" --model M override backend default model") print(" --mode deep aggressive INFERRED-edge semantic extraction") + print(" --force full re-scan and re-dispatch: skip the incremental") + print(" manifest gate and semantic cache reads (env: GRAPHIFY_FORCE=1)") print(" --max-workers N AST extraction subprocess count (default: cpu_count)") print(" --token-budget N per-chunk token cap for semantic extraction (default: 60000)") print(" --max-concurrency N parallel semantic chunks in flight (default: 4; set 1 for local LLMs)") print(" --api-timeout S per-request timeout in seconds for the LLM client (default: 600)") - print(" --out DIR output dir (default: ); writes

/graphify-out/") + print(" --out DIR, --output DIR output dir (default: ); writes /graphify-out/") print(" --google-workspace export .gdoc/.gsheet/.gslides shortcuts via gws before extraction") + print(" --no-gitignore ignore .gitignore and .git/info/exclude (prioritizes .graphifyignore)") print(" --no-cluster skip clustering, write raw extraction only") + print(" --code-only index code (local AST, no API key) and skip doc/paper/image files") print(" --postgres DSN extract schema from a live PostgreSQL database") print(" maps tables, views, functions + FK relationships;") print(" column-level detail is not represented in the graph") @@ -2453,2750 +707,9 @@ def main() -> None: print(f"Run 'graphify --help' for full usage.") return - if cmd == "install": - # Default to windows platform on Windows, claude elsewhere - default_platform = "windows" if platform.system() == "Windows" else "claude" - selected_platform: str | None = None - project_scope = False - args = sys.argv[2:] - i = 0 - while i < len(args): - arg = args[i] - if arg in ("-h", "--help"): - _print_install_usage() - return - if arg == "--project": - project_scope = True - i += 1 - elif arg.startswith("--platform="): - candidate = arg.split("=", 1)[1] - if selected_platform and selected_platform != candidate: - print("error: specify install platform only once", file=sys.stderr) - sys.exit(1) - selected_platform = candidate - i += 1 - elif arg == "--platform": - if i + 1 >= len(args): - print("error: --platform requires a value", file=sys.stderr) - sys.exit(1) - candidate = args[i + 1] - if selected_platform and selected_platform != candidate: - print("error: specify install platform only once", file=sys.stderr) - sys.exit(1) - selected_platform = candidate - i += 2 - elif arg.startswith("-"): - print(f"error: unknown install option '{arg}'", file=sys.stderr) - sys.exit(1) - else: - if selected_platform and selected_platform != arg: - print("error: specify install platform only once", file=sys.stderr) - sys.exit(1) - selected_platform = arg - i += 1 - chosen_platform = selected_platform or default_platform - if project_scope: - _project_install(chosen_platform, Path(".")) - else: - install(platform=chosen_platform) - elif cmd == "uninstall": - args = sys.argv[2:] - purge = "--purge" in args - project_scope = "--project" in args - selected_platform = None - i = 0 - while i < len(args): - arg = args[i] - if arg in ("--purge", "--project"): - i += 1 - elif arg.startswith("--platform="): - selected_platform = arg.split("=", 1)[1] - i += 1 - elif arg == "--platform": - if i + 1 >= len(args): - print("error: --platform requires a value", file=sys.stderr) - sys.exit(1) - selected_platform = args[i + 1] - i += 2 - elif arg.startswith("-"): - print(f"error: unknown uninstall option '{arg}'", file=sys.stderr) - sys.exit(1) - else: - selected_platform = arg - i += 1 - if project_scope: - if selected_platform: - _project_uninstall(selected_platform, Path(".")) - else: - _project_uninstall_all(Path(".")) - else: - uninstall_all(purge=purge) - elif cmd == "claude": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - if "--project" in sys.argv[3:]: - _project_install("claude", Path(".")) - else: - claude_install() - elif subcmd == "uninstall": - if "--project" in sys.argv[3:]: - _project_uninstall("claude", Path(".")) - else: - claude_uninstall() - else: - print("Usage: graphify claude [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "codebuddy": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - codebuddy_install() - elif subcmd == "uninstall": - codebuddy_uninstall() - else: - print("Usage: graphify codebuddy [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "gemini": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - gemini_install(project=("--project" in sys.argv[3:])) - elif subcmd == "uninstall": - gemini_uninstall(project=("--project" in sys.argv[3:])) - else: - print("Usage: graphify gemini [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "cursor": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - _cursor_install(Path(".")) - elif subcmd == "uninstall": - _cursor_uninstall(Path(".")) - else: - print("Usage: graphify cursor [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "vscode": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - vscode_install() - elif subcmd == "uninstall": - vscode_uninstall() - else: - print("Usage: graphify vscode [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "copilot": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - if "--project" in sys.argv[3:]: - _project_install("copilot", Path(".")) - else: - install(platform="copilot") - elif subcmd == "uninstall": - if "--project" in sys.argv[3:]: - _project_uninstall("copilot", Path(".")) - else: - removed = _remove_skill_file("copilot") - print("skill removed" if removed else "nothing to remove") - else: - print("Usage: graphify copilot [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "kilo": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - _kilo_install(Path(".")) - elif subcmd == "uninstall": - _kilo_uninstall(Path(".")) - else: - print("Usage: graphify kilo [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "kiro": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - _kiro_install(Path(".")) - elif subcmd == "uninstall": - _kiro_uninstall(Path(".")) - else: - print("Usage: graphify kiro [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "devin": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - if "--project" in sys.argv[3:]: - _project_install("devin", Path(".")) - else: - install(platform="devin") - elif subcmd == "uninstall": - if "--project" in sys.argv[3:]: - _project_uninstall("devin", Path(".")) - else: - removed = _remove_skill_file("devin") - print("skill removed" if removed else "nothing to remove") - else: - print("Usage: graphify devin [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "pi": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - if "--project" in sys.argv[3:]: - _project_install("pi", Path(".")) - else: - install("pi") - elif subcmd == "uninstall": - if "--project" in sys.argv[3:]: - _project_uninstall("pi", Path(".")) - else: - _remove_skill_file("pi") - else: - print("Usage: graphify pi [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "amp": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - if "--project" in sys.argv[3:]: - _project_install("amp", Path(".")) - else: - _amp_install(Path(".")) - elif subcmd == "uninstall": - if "--project" in sys.argv[3:]: - _project_uninstall("amp", Path(".")) - else: - _amp_uninstall(Path(".")) - else: - print("Usage: graphify amp [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd in ("agents", "skills"): - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - if "--project" in sys.argv[3:]: - _project_install("agents", Path(".")) - else: - _agents_platform_install(Path(".")) - elif subcmd == "uninstall": - if "--project" in sys.argv[3:]: - _project_uninstall("agents", Path(".")) - else: - _agents_platform_uninstall(Path(".")) - else: - print(f"Usage: graphify {cmd} [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd in ("aider", "codex", "opencode", "claw", "droid", "trae", "trae-cn", "hermes"): - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - if "--project" in sys.argv[3:]: - _project_install(cmd, Path(".")) - else: - _agents_install(Path("."), cmd) - elif subcmd == "uninstall": - if "--project" in sys.argv[3:]: - _project_uninstall(cmd, Path(".")) - else: - _agents_uninstall(Path("."), platform=cmd) - if cmd == "codex": - _uninstall_codex_hook(Path(".")) - else: - print(f"Usage: graphify {cmd} [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "antigravity": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - if "--project" in sys.argv[3:]: - _project_install("antigravity", Path(".")) - else: - _antigravity_install(Path(".")) - elif subcmd == "uninstall": - if "--project" in sys.argv[3:]: - _project_uninstall("antigravity", Path(".")) - else: - _antigravity_uninstall(Path(".")) - else: - print("Usage: graphify antigravity [install|uninstall]", file=sys.stderr) - sys.exit(1) - elif cmd == "provider": - from graphify.llm import _custom_providers_path, BACKENDS - import json as _json - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - global_path = _custom_providers_path(global_=True) - - if subcmd == "list": - global_path.parent.mkdir(parents=True, exist_ok=True) - existing: dict = {} - if global_path.is_file(): - try: - existing = _json.loads(global_path.read_text(encoding="utf-8")) - except Exception: - pass - if not existing: - print("No custom providers registered.") - else: - for name in existing: - print(f" {name} ({existing[name].get('base_url', '')})") - - elif subcmd == "show": - name = sys.argv[3] if len(sys.argv) > 3 else "" - if not name: - print("Usage: graphify provider show ", file=sys.stderr) - sys.exit(1) - existing = {} - if global_path.is_file(): - try: - existing = _json.loads(global_path.read_text(encoding="utf-8")) - except Exception: - pass - if name not in existing: - print(f"Provider '{name}' not found.", file=sys.stderr) - sys.exit(1) - print(_json.dumps({name: existing[name]}, indent=2)) - - elif subcmd == "add": - args = sys.argv[3:] - name = args[0] if args and not args[0].startswith("-") else "" - if not name: - print("Usage: graphify provider add --base-url URL --default-model MODEL --env-key KEY", file=sys.stderr) - sys.exit(1) - if name in BACKENDS: - print(f"Error: '{name}' is a built-in provider and cannot be overridden.", file=sys.stderr) - sys.exit(1) - base_url = "" - default_model = "" - env_key = "" - pricing_input = 0.0 - pricing_output = 0.0 - i = 1 - while i < len(args): - a = args[i] - if a == "--base-url" and i + 1 < len(args): - base_url = args[i + 1]; i += 2 - elif a.startswith("--base-url="): - base_url = a.split("=", 1)[1]; i += 1 - elif a == "--default-model" and i + 1 < len(args): - default_model = args[i + 1]; i += 2 - elif a.startswith("--default-model="): - default_model = a.split("=", 1)[1]; i += 1 - elif a == "--env-key" and i + 1 < len(args): - env_key = args[i + 1]; i += 2 - elif a.startswith("--env-key="): - env_key = a.split("=", 1)[1]; i += 1 - elif a == "--pricing-input" and i + 1 < len(args): - pricing_input = float(args[i + 1]); i += 2 - elif a == "--pricing-output" and i + 1 < len(args): - pricing_output = float(args[i + 1]); i += 2 - else: - i += 1 - if not base_url or not default_model or not env_key: - print("Error: --base-url, --default-model, and --env-key are required.", file=sys.stderr) - sys.exit(1) - from graphify.llm import provider_base_url_ok - if not provider_base_url_ok(base_url, name): - print(f"Error: refusing to add provider with unsafe base_url {base_url!r}.", file=sys.stderr) - sys.exit(1) - global_path.parent.mkdir(parents=True, exist_ok=True) - existing = {} - if global_path.is_file(): - try: - existing = _json.loads(global_path.read_text(encoding="utf-8")) - except Exception: - pass - existing[name] = { - "base_url": base_url, - "default_model": default_model, - "env_key": env_key, - "pricing": {"input": pricing_input, "output": pricing_output}, - "temperature": 0, - } - global_path.write_text(_json.dumps(existing, indent=2) + "\n", encoding="utf-8") - print(f"Provider '{name}' added. Use with: graphify extract . --backend {name}") - - elif subcmd == "remove": - name = sys.argv[3] if len(sys.argv) > 3 else "" - if not name: - print("Usage: graphify provider remove ", file=sys.stderr) - sys.exit(1) - existing = {} - if global_path.is_file(): - try: - existing = _json.loads(global_path.read_text(encoding="utf-8")) - except Exception: - pass - if name not in existing: - print(f"Provider '{name}' not found.", file=sys.stderr) - sys.exit(1) - del existing[name] - global_path.write_text(_json.dumps(existing, indent=2) + "\n", encoding="utf-8") - print(f"Provider '{name}' removed.") - - else: - print("Usage: graphify provider [add|list|show|remove]", file=sys.stderr) - if subcmd: - sys.exit(1) - elif cmd == "prs": - from graphify.prs import cmd_prs - cmd_prs(sys.argv[2:]) - elif cmd == "hook": - from graphify.hooks import ( - install as hook_install, - uninstall as hook_uninstall, - status as hook_status, - ) - - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd == "install": - print(hook_install(Path("."))) - elif subcmd == "uninstall": - print(hook_uninstall(Path("."))) - elif subcmd == "status": - print(hook_status(Path("."))) - else: - print("Usage: graphify hook [install|uninstall|status]", file=sys.stderr) - sys.exit(1) - elif cmd == "query": - if len(sys.argv) < 3: - print("Usage: graphify query \"\" [--dfs] [--context C] [--budget N] [--graph path]", file=sys.stderr) - sys.exit(1) - from graphify.serve import _query_graph_text - from graphify.security import sanitize_label - from networkx.readwrite import json_graph - from graphify import querylog - - question = sys.argv[2] - use_dfs = "--dfs" in sys.argv - budget = 2000 - graph_path = _default_graph_path() - context_filters: list[str] = [] - args = sys.argv[3:] - i = 0 - while i < len(args): - if args[i] == "--budget" and i + 1 < len(args): - try: - budget = int(args[i + 1]) - except ValueError: - print(f"error: --budget must be an integer", file=sys.stderr) - sys.exit(1) - i += 2 - elif args[i].startswith("--budget="): - try: - budget = int(args[i].split("=", 1)[1]) - except ValueError: - print(f"error: --budget must be an integer", file=sys.stderr) - sys.exit(1) - i += 1 - elif args[i] == "--context" and i + 1 < len(args): - context_filters.append(args[i + 1]) - i += 2 - elif args[i].startswith("--context="): - context_filters.append(args[i].split("=", 1)[1]) - i += 1 - elif args[i] == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - i += 2 - else: - i += 1 - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - if not gp.suffix == ".json": - print(f"error: graph file must be a .json file", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - try: - import json as _json - import networkx as _nx - - _raw = _json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) - try: - from graphify.build import graph_has_legacy_ids as _legacy - if _legacy(_raw.get("nodes", [])): - print( - "[graphify] note: this graph uses the pre-#1504 node-ID scheme; " - "rebuild with `graphify extract --force` to get path-qualified IDs " - "(fixes same-name-file collisions).", - file=sys.stderr, - ) - except Exception: - pass - except Exception as exc: - print(f"error: could not load graph: {exc}", file=sys.stderr) - sys.exit(1) - import time as _time - _t0 = _time.perf_counter() - _mode = "dfs" if use_dfs else "bfs" - _result = _query_graph_text( - G, - question, - mode=_mode, - depth=2, - token_budget=budget, - context_filters=context_filters, - ) - querylog.log_query( - kind="query", - question=question, - corpus=str(gp), - result=_result, - mode=_mode, - depth=2, - token_budget=budget, - duration_ms=(_time.perf_counter() - _t0) * 1000, - ) - print(_result) - elif cmd == "affected": - if len(sys.argv) < 3: - print("Usage: graphify affected \"\" [--relation R] [--depth N] [--graph path]", file=sys.stderr) - sys.exit(1) - from graphify.affected import DEFAULT_AFFECTED_RELATIONS, format_affected, load_graph - query = sys.argv[2] - graph_path = _default_graph_path() - depth = 2 - relations: list[str] = [] - args = sys.argv[3:] - i = 0 - while i < len(args): - if args[i] == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - i += 2 - elif args[i].startswith("--graph="): - graph_path = args[i].split("=", 1)[1] - i += 1 - elif args[i] == "--depth" and i + 1 < len(args): - try: - depth = int(args[i + 1]) - except ValueError: - print("error: --depth must be an integer", file=sys.stderr) - sys.exit(1) - i += 2 - elif args[i].startswith("--depth="): - try: - depth = int(args[i].split("=", 1)[1]) - except ValueError: - print("error: --depth must be an integer", file=sys.stderr) - sys.exit(1) - i += 1 - elif args[i] == "--relation" and i + 1 < len(args): - relations.append(args[i + 1]) - i += 2 - elif args[i].startswith("--relation="): - relations.append(args[i].split("=", 1)[1]) - i += 1 - else: - i += 1 - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - if not gp.suffix == ".json": - print("error: graph file must be a .json file", file=sys.stderr) - sys.exit(1) - try: - graph = load_graph(gp) - except Exception as exc: - print(f"error: could not load graph: {exc}", file=sys.stderr) - sys.exit(1) - print( - format_affected( - graph, - query, - relations=relations or DEFAULT_AFFECTED_RELATIONS, - depth=depth, - ) - ) - elif cmd == "save-result": - # graphify save-result --question Q --answer A [--type T] [--nodes N1 N2 ...] - # [--outcome useful|dead_end|corrected] [--correction TEXT] - import argparse as _ap - - p = _ap.ArgumentParser(prog="graphify save-result") - p.add_argument("--question", required=True) - p.add_argument("--answer", default=None) - p.add_argument("--answer-file", dest="answer_file", default=None) - p.add_argument("--type", dest="query_type", default="query") - p.add_argument("--nodes", nargs="*", default=[]) - p.add_argument("--outcome", choices=("useful", "dead_end", "corrected"), default=None) - p.add_argument("--correction", default=None) - p.add_argument("--memory-dir", default=str(Path(_GRAPHIFY_OUT) / "memory")) - opts = p.parse_args(sys.argv[2:]) - if opts.answer_file: - opts.answer = Path(opts.answer_file).read_text(encoding="utf-8").strip() - elif not opts.answer: - p.error("--answer or --answer-file is required") - from graphify.ingest import save_query_result as _sqr - - out = _sqr( - question=opts.question, - answer=opts.answer, - memory_dir=Path(opts.memory_dir), - query_type=opts.query_type, - source_nodes=opts.nodes or None, - outcome=opts.outcome, - correction=opts.correction, - ) - print(f"Saved to {out}") - elif cmd == "reflect": - import argparse as _ap - - p = _ap.ArgumentParser(prog="graphify reflect") - p.add_argument("--memory-dir", default=str(Path(_GRAPHIFY_OUT) / "memory")) - p.add_argument( - "--out", - default=str(Path(_GRAPHIFY_OUT) / "reflections" / "LESSONS.md"), - ) - p.add_argument("--graph", default=None) - p.add_argument("--analysis", default=None) - p.add_argument("--labels", default=None) - p.add_argument("--half-life-days", type=float, default=30.0, - help="signal weight halves every N days (default 30)") - p.add_argument("--min-corroboration", type=int, default=2, - help="distinct useful results to promote a node to preferred (default 2)") - p.add_argument("--if-stale", action="store_true", - help="skip when LESSONS.md is already newer than every input " - "(e.g. the git hook just refreshed it)") - opts = p.parse_args(sys.argv[2:]) - from graphify.reflect import reflect as _reflect, lessons_fresh as _lessons_fresh - - graph_arg = opts.graph - if graph_arg is None: - default_graph = Path(_GRAPHIFY_OUT) / "graph.json" - if default_graph.exists(): - graph_arg = str(default_graph) - - _gp = Path(graph_arg) if graph_arg else None - _analysis_path = None - _labels_path = None - if _gp is not None: - _analysis_path = Path(opts.analysis) if opts.analysis else ( - _gp.parent / ".graphify_analysis.json") - _labels_path = Path(opts.labels) if opts.labels else ( - _gp.parent / ".graphify_labels.json") - - if opts.if_stale and _lessons_fresh( - Path(opts.out), Path(opts.memory_dir), _gp, _analysis_path, _labels_path - ): - print(f"Lessons already up to date -> {opts.out} (skipped; omit --if-stale to force)") - else: - out_path, agg = _reflect( - memory_dir=Path(opts.memory_dir), - out_path=Path(opts.out), - graph_path=_gp, - analysis_path=_analysis_path, - labels_path=_labels_path, - half_life_days=opts.half_life_days, - min_corroboration=opts.min_corroboration, - ) - c = agg["counts"] - print( - f"Reflected {agg['total']} memories " - f"({c['useful']} useful, {c['dead_end']} dead ends, " - f"{c['corrected']} corrected) -> {out_path}" - ) - elif cmd == "path": - if len(sys.argv) < 4: - print( - 'Usage: graphify path "" "" [--graph path]', - file=sys.stderr, - ) - sys.exit(1) - from graphify.serve import _score_nodes - from networkx.readwrite import json_graph - import networkx as _nx - - source_label = sys.argv[2] - target_label = sys.argv[3] - graph_path = _default_graph_path() - args = sys.argv[4:] - for i, a in enumerate(args): - if a == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - _raw = json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - # Force directed so the renderer can recover stored caller→callee direction. - _raw = {**_raw, "directed": True} - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) - src_scored = _score_nodes(G, [t.lower() for t in source_label.split()]) - tgt_scored = _score_nodes(G, [t.lower() for t in target_label.split()]) - if not src_scored: - print(f"No node matching '{source_label}' found.", file=sys.stderr) - sys.exit(1) - if not tgt_scored: - print(f"No node matching '{target_label}' found.", file=sys.stderr) - sys.exit(1) - src_nid, tgt_nid = src_scored[0][1], tgt_scored[0][1] - # Ambiguity guard: when both queries resolve to the same node, the - # shortest path is trivially zero hops, which is almost never what the - # caller wanted (see bug #828). - if src_nid == tgt_nid: - print( - f"'{source_label}' and '{target_label}' both resolved to the same " - f"node '{src_nid}'. Use a more specific label or the exact node ID.", - file=sys.stderr, - ) - sys.exit(1) - for _name, _scored in (("source", src_scored), ("target", tgt_scored)): - if len(_scored) >= 2: - _top, _runner = _scored[0][0], _scored[1][0] - if _top > 0 and (_top - _runner) / _top < 0.10: - print( - f"warning: {_name} match was ambiguous " - f"(top score {_top:g}, runner-up {_runner:g})", - file=sys.stderr, - ) - try: - path_nodes = _nx.shortest_path(G.to_undirected(as_view=True), src_nid, tgt_nid) - except (_nx.NetworkXNoPath, _nx.NodeNotFound): - print(f"No path found between '{source_label}' and '{target_label}'.") - sys.exit(0) - hops = len(path_nodes) - 1 - segments = [] - from graphify.build import edge_data - for i in range(len(path_nodes) - 1): - u, v = path_nodes[i], path_nodes[i + 1] - # Check which direction the stored edge points. - if G.has_edge(u, v): - edata = edge_data(G, u, v) - forward = True - else: - edata = edge_data(G, v, u) - forward = False - rel = edata.get("relation", "") - conf = edata.get("confidence", "") - conf_str = f" [{conf}]" if conf else "" - if i == 0: - segments.append(G.nodes[u].get("label", u)) - if forward: - segments.append(f"--{rel}{conf_str}--> {G.nodes[v].get('label', v)}") - else: - segments.append(f"<--{rel}{conf_str}-- {G.nodes[v].get('label', v)}") - print(f"Shortest path ({hops} hops):\n " + " ".join(segments)) - from graphify import querylog - querylog.log_query( - kind="path", - question=f"{sys.argv[2]} -> {sys.argv[3]}", - corpus=str(gp), - nodes_returned=hops, - ) - - elif cmd == "explain": - if len(sys.argv) < 3: - print('Usage: graphify explain "" [--graph path]', file=sys.stderr) - sys.exit(1) - from graphify.serve import _find_node - from networkx.readwrite import json_graph - - label = sys.argv[2] - graph_path = _default_graph_path() - args = sys.argv[3:] - for i, a in enumerate(args): - if a == "--graph" and i + 1 < len(args): - graph_path = args[i + 1] - gp = Path(graph_path).resolve() - if not gp.exists(): - print(f"error: graph file not found: {gp}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - _raw = json.loads(gp.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - # Force directed so the renderer can recover stored caller→callee direction. - _raw = {**_raw, "directed": True} - try: - G = json_graph.node_link_graph(_raw, edges="links") - except TypeError: - G = json_graph.node_link_graph(_raw) - matches = _find_node(G, label) - if not matches: - print(f"No node matching '{label}' found.") - sys.exit(0) - nid = matches[0] - d = G.nodes[nid] - print(f"Node: {d.get('label', nid)}") - print(f" ID: {nid}") - print( - f" Source: {d.get('source_file', '')} {d.get('source_location', '')}".rstrip() - ) - print(f" Type: {d.get('file_type', '')}") - print(f" Community: {d.get('community_name') or d.get('community', '')}") - # Work-memory overlay: a derived experiential hint from `graphify reflect`, - # merged in display-only from the .graphify_learning.json sidecar next to - # graph.json. No line when the node has no overlay entry. - try: - from graphify.reflect import load_learning_overlay as _llo - from graphify.security import sanitize_label as _sl - _overlay = _llo(gp) - _entry = _overlay.get(str(nid)) - if _entry: - _status = _sl(str(_entry.get("status", ""))) - if _status == "contested": - _line = (f" Lesson: contested (useful {_entry.get('uses', 0)} / " - f"dead-end {_entry.get('neg', 0)})") - elif _status == "preferred": - _line = (f" Lesson: preferred source (start here) — " - f"{_entry.get('uses', 0)} useful, score={_entry.get('score', 0)}") - else: - _line = (f" Lesson: {_status or 'tentative'} — " - f"{_entry.get('uses', 0)} useful, score={_entry.get('score', 0)}") - if _entry.get("stale"): - _line += " [code changed since — re-verify]" - print(_line) - except Exception: - pass - print(f" Degree: {G.degree(nid)}") - from graphify.build import edge_data - connections: list[tuple[str, str, dict]] = [] # (direction, neighbor_id, edge_data) - for nb in G.successors(nid): - connections.append(("out", nb, edge_data(G, nid, nb))) - for nb in G.predecessors(nid): - connections.append(("in", nb, edge_data(G, nb, nid))) - if connections: - print(f"\nConnections ({len(connections)}):") - connections.sort(key=lambda c: G.degree(c[1]), reverse=True) - for direction, nb, edata in connections[:20]: - rel = edata.get("relation", "") - conf = edata.get("confidence", "") - arrow = "-->" if direction == "out" else "<--" - print(f" {arrow} {G.nodes[nb].get('label', nb)} [{rel}] [{conf}]") - if len(connections) > 20: - print(f" ... and {len(connections) - 20} more") - from graphify import querylog - querylog.log_query( - kind="explain", - question=sys.argv[2], - corpus=str(gp), - nodes_returned=len(connections), - ) - - elif cmd == "diagnose": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd != "multigraph": - print( - "Usage: graphify diagnose multigraph " - "[--graph path] [--json] [--max-examples N] " - "[--directed] [--undirected] [--extract-path path]", - file=sys.stderr, - ) - sys.exit(1) - - graph_path = Path(_default_graph_path()) - max_examples = 5 - directed: bool | None = None - direction_flag: str | None = None - json_output = False - extract_path: Path | None = None - - i = 3 - while i < len(sys.argv): - arg = sys.argv[i] - if arg == "--graph": - i += 1 - if i >= len(sys.argv): - print("error: --graph requires a path", file=sys.stderr) - sys.exit(1) - graph_path = Path(sys.argv[i]) - elif arg == "--json": - json_output = True - elif arg == "--max-examples": - i += 1 - if i >= len(sys.argv): - print("error: --max-examples requires an integer", file=sys.stderr) - sys.exit(1) - try: - max_examples = int(sys.argv[i]) - except ValueError: - print("error: --max-examples requires an integer", file=sys.stderr) - sys.exit(1) - if max_examples < 0: - print("error: --max-examples must be >= 0", file=sys.stderr) - sys.exit(1) - elif arg == "--directed": - if direction_flag == "undirected": - print( - "error: --directed and --undirected are mutually exclusive", - file=sys.stderr, - ) - sys.exit(1) - direction_flag = "directed" - directed = True - elif arg == "--undirected": - if direction_flag == "directed": - print( - "error: --directed and --undirected are mutually exclusive", - file=sys.stderr, - ) - sys.exit(1) - direction_flag = "undirected" - directed = False - elif arg == "--extract-path": - i += 1 - if i >= len(sys.argv): - print("error: --extract-path requires a path", file=sys.stderr) - sys.exit(1) - extract_path = Path(sys.argv[i]) - else: - print(f"error: unknown diagnose option {arg}", file=sys.stderr) - sys.exit(1) - i += 1 - - from graphify.diagnostics import ( - diagnose_file, - format_diagnostic_json, - format_diagnostic_report, - ) - - try: - summary = diagnose_file( - graph_path, - directed=directed, - root=Path(".").resolve(), - max_examples=max_examples, - extract_path=extract_path, - ) - except Exception as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - - if json_output: - print(json.dumps(format_diagnostic_json(summary), indent=2)) - else: - print(format_diagnostic_report(summary)) - - elif cmd == "add": - if len(sys.argv) < 3: - print( - "Usage: graphify add [--author Name] [--contributor Name] [--dir ./raw]", - file=sys.stderr, - ) - sys.exit(1) - from graphify.ingest import ingest as _ingest - - url = sys.argv[2] - author: str | None = None - contributor: str | None = None - target_dir = Path("raw") - args = sys.argv[3:] - i = 0 - while i < len(args): - if args[i] == "--author" and i + 1 < len(args): - author = args[i + 1] - i += 2 - elif args[i] == "--contributor" and i + 1 < len(args): - contributor = args[i + 1] - i += 2 - elif args[i] == "--dir" and i + 1 < len(args): - target_dir = Path(args[i + 1]) - i += 2 - else: - i += 1 - try: - saved = _ingest(url, target_dir, author=author, contributor=contributor) - print(f"Saved to {saved}") - print("Run /graphify --update in your AI assistant to update the graph.") - except Exception as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - - elif cmd == "watch": - watch_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path(".") - if not watch_path.exists(): - print(f"error: path not found: {watch_path}", file=sys.stderr) - sys.exit(1) - from graphify.watch import watch as _watch - - try: - _watch(watch_path) - except ImportError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - - elif cmd in ("cluster-only", "label"): - # `label` is `cluster-only` that always (re)generates community names with - # the configured backend, even when a .graphify_labels.json already exists. - force_relabel = cmd == "label" - # Mirror the tree/export arg-parsing pattern: walk argv so flags and - # the optional positional path can appear in any order (#724). - no_viz = "--no-viz" in sys.argv - no_label = "--no-label" in sys.argv - missing_only = "--missing-only" in sys.argv - co_timing = "--timing" in sys.argv - _backend_arg = next((a for a in sys.argv if a.startswith("--backend=")), None) - label_backend = _backend_arg.split("=", 1)[1] if _backend_arg else None - _model_arg = next((a for a in sys.argv if a.startswith("--model=")), None) - label_model = _model_arg.split("=", 1)[1] if _model_arg else None - _min_cs_arg = next((a for a in sys.argv if a.startswith("--min-community-size=")), None) - min_community_size = int(_min_cs_arg.split("=")[1]) if _min_cs_arg else 3 - args = sys.argv[2:] - watch_path: Path | None = None - graph_override: Path | None = None - co_resolution: float = 1.0 - co_exclude_hubs: float | None = None - label_max_concurrency: int = 4 - label_batch_size: int = 100 - i_arg = 0 - while i_arg < len(args): - a = args[i_arg] - if a == "--graph" and i_arg + 1 < len(args): - graph_override = Path(args[i_arg + 1]); i_arg += 2 - elif a == "--backend" and i_arg + 1 < len(args): - label_backend = args[i_arg + 1]; i_arg += 2 - elif a.startswith("--backend="): - label_backend = a.split("=", 1)[1]; i_arg += 1 - elif a == "--model" and i_arg + 1 < len(args): - label_model = args[i_arg + 1]; i_arg += 2 - elif a.startswith("--model="): - label_model = a.split("=", 1)[1]; i_arg += 1 - elif a == "--resolution" and i_arg + 1 < len(args): - co_resolution = float(args[i_arg + 1]); i_arg += 2 - elif a.startswith("--resolution="): - co_resolution = float(a.split("=", 1)[1]); i_arg += 1 - elif a == "--exclude-hubs" and i_arg + 1 < len(args): - co_exclude_hubs = float(args[i_arg + 1]); i_arg += 2 - elif a.startswith("--exclude-hubs="): - co_exclude_hubs = float(a.split("=", 1)[1]); i_arg += 1 - elif a == "--max-concurrency" and i_arg + 1 < len(args): - label_max_concurrency = int(args[i_arg + 1]); i_arg += 2 - elif a.startswith("--max-concurrency="): - label_max_concurrency = int(a.split("=", 1)[1]); i_arg += 1 - elif a == "--batch-size" and i_arg + 1 < len(args): - label_batch_size = int(args[i_arg + 1]); i_arg += 2 - elif a.startswith("--batch-size="): - label_batch_size = int(a.split("=", 1)[1]); i_arg += 1 - elif a in ("--no-viz", "--missing-only") or a.startswith("--min-community-size="): - i_arg += 1 - elif a.startswith("--"): - i_arg += 1 - elif watch_path is None: - watch_path = Path(a); i_arg += 1 - else: - i_arg += 1 - if watch_path is None: - watch_path = Path(".") - graph_json = graph_override if graph_override is not None else watch_path / _GRAPHIFY_OUT / "graph.json" - if not graph_json.exists(): - print( - f"error: no graph found at {graph_json} — run /graphify first", - file=sys.stderr, - ) - sys.exit(1) - from networkx.readwrite import json_graph as _jg - from graphify.build import build_from_json - from graphify.cluster import cluster, score_all, remap_communities_to_previous - from graphify.analyze import ( - god_nodes, - surprising_connections, - suggest_questions, - ) - from graphify.report import generate - from graphify.export import to_json, to_html - - stages = _StageTimer(co_timing) - print("Loading existing graph...") - # Solution 3 (#1019): don't hard-exit on an oversized graph.json here. - # Core outputs (graph.json + GRAPH_REPORT.md) still get written; the - # graph.html render below falls back to the community-aggregation view - # (node_limit=5000) when over the cap. - from graphify.security import check_graph_file_size_cap as _check_cap - _over_cap = False - try: - _check_cap(graph_json) - except ValueError: - _over_cap = True - try: - _over_cap_bytes = graph_json.stat().st_size - except OSError: - _over_cap_bytes = -1 - print( - f"warning: graph.json exceeds cap ({_over_cap_bytes} bytes); " - f"falling back to community-aggregation view (node_limit=5000)", - file=sys.stderr, - ) - _raw = json.loads(graph_json.read_text(encoding="utf-8")) - _directed = bool(_raw.get("directed", False)) - G = build_from_json(_raw, directed=_directed) - print(f"Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") - stages.mark("load") - print("Re-clustering...") - communities = cluster(G, resolution=co_resolution, exclude_hubs_percentile=co_exclude_hubs) - # Mirror the watch/update path (#822): map new cids to prior ones by - # node-overlap so the existing .graphify_labels.json keeps attaching - # to the same conceptual community after re-clustering. Without this, - # labels follow raw cid index and become misaligned whenever the - # graph has changed between labeling and cluster-only (#1027). - previous_node_community = { - n["id"]: n["community"] - for n in _raw.get("nodes", []) - if n.get("community") is not None and n.get("id") is not None - } - if previous_node_community: - communities = remap_communities_to_previous(communities, previous_node_community) - stages.mark("cluster") - cohesion = score_all(G, communities) - gods = god_nodes(G) - surprises = surprising_connections(G, communities) - stages.mark("analyze") - out = watch_path / _GRAPHIFY_OUT - out.mkdir(parents=True, exist_ok=True) - labels_path = out / ".graphify_labels.json" - existing_labels: dict[int, str] = {} - if labels_path.exists(): - try: - existing_labels = { - int(k): v - for k, v in json.loads(labels_path.read_text(encoding="utf-8")).items() - if isinstance(v, str) - } - except Exception: - existing_labels = {} - if labels_path.exists() and not force_relabel: - # Reuse saved labels, but don't blindly trust them: the graph may have - # been re-scoped/re-clustered since labeling, in which case a cid now - # covers a DIFFERENT community and its old (LLM) name is wrong (#label-stale). - # Validate each community against the membership signature saved beside the - # labels; any community that changed (or has no saved label) is renamed by - # its current hub — deterministic and correct-by-construction — and the user - # is told to `graphify label` for fresh LLM names. Unchanged communities keep - # their saved label. When no signature sidecar exists (labels predate this), - # fall back to hub-filling only the communities missing a label. - from graphify.cluster import community_member_sigs, label_communities_by_hub - sig_path = labels_path.parent / (labels_path.name + ".sig") - saved_sigs: dict[int, str] = {} - if sig_path.exists(): - try: - saved_sigs = { - int(k): v for k, v in - json.loads(sig_path.read_text(encoding="utf-8")).items() - if isinstance(v, str) - } - except Exception: - saved_sigs = {} - cur_sigs = community_member_sigs(communities) - count_mismatch = len(existing_labels) != len(communities) - labels = {} - hub_labels: dict[int, str] | None = None - changed = 0 - for cid in communities: - have_label = cid in existing_labels - if saved_sigs: - # Precise: the membership signature tells us if this exact - # community changed since it was labeled. - fresh = have_label and saved_sigs.get(cid) == cur_sigs.get(cid) - else: - # No signature sidecar (labels predate it). A differing community - # COUNT means the labels describe a different clustering, so a cid's - # old label can't be trusted; equal count is the best "same" signal. - fresh = have_label and not count_mismatch - if fresh: - labels[cid] = existing_labels[cid] - else: - if hub_labels is None: - hub_labels = label_communities_by_hub(G, communities) - labels[cid] = hub_labels[cid] - if have_label: - changed += 1 - if changed: - print( - f"[graphify] community set changed since labeling " - f"({len(existing_labels)} saved labels, {len(communities)} communities now; " - f"renamed {changed} community(ies) by their hub). " - f"Run `graphify label` to refresh names with the LLM.", - file=sys.stderr, - ) - elif no_label and not force_relabel: - labels = {cid: f"Community {cid}" for cid in communities} - else: - # No labels file yet (or `graphify label` forced a refresh). When run - # standalone there is no orchestrating agent to do skill.md Step 5, so - # auto-name communities rather than leave "Community N" (#1097). - from graphify.cluster import label_communities_by_hub - from graphify.llm import generate_community_labels - print("Labeling communities...") - # Deterministic, LLM-free base labels: name each community after its - # highest-degree hub, so the report is readable even with no backend - # (previously bare "Community N"). A configured LLM backend overrides these - # with richer names below; its no-backend placeholder fallback does NOT. - hub_labels = label_communities_by_hub(G, communities) - label_communities_input = communities - labels = dict(hub_labels) - if missing_only: - labels = { - cid: existing_labels.get(cid, hub_labels[cid]) - for cid in communities - } - label_communities_input = { - cid: members - for cid, members in communities.items() - if cid not in existing_labels or existing_labels.get(cid) == f"Community {cid}" - } - generated_labels, _ = generate_community_labels( - G, label_communities_input, backend=label_backend, model=label_model, gods=gods, - max_concurrency=label_max_concurrency, batch_size=label_batch_size, - ) - # Only let the LLM OVERRIDE where it produced a real name — its no-backend - # fallback returns "Community {cid}" placeholders, which must not clobber - # the deterministic hub labels. - labels.update({ - cid: v for cid, v in generated_labels.items() - if v and v != f"Community {cid}" - }) - stages.mark("label") - questions = suggest_questions(G, communities, labels) - tokens = {"input": 0, "output": 0} - from graphify.export import _git_head as _gh - _commit = _gh() - from graphify.report import load_learning_for_report as _llfr - report = generate(G, communities, cohesion, labels, gods, surprises, - {"warning": "cluster-only mode — file stats not available"}, - tokens, str(watch_path), suggested_questions=questions, - min_community_size=min_community_size, built_at_commit=_commit, - learning=_llfr(out / "graph.json")) - (out / "GRAPH_REPORT.md").write_text(report, encoding="utf-8") - stages.mark("report") - from graphify.export import backup_if_protected as _backup - _backup(out) - analysis = { - "communities": {str(k): v for k, v in communities.items()}, - "cohesion": {str(k): v for k, v in cohesion.items()}, - "gods": gods, - "surprises": surprises, - "questions": questions, - } - (out / ".graphify_analysis.json").write_text( - json.dumps(analysis, indent=2, ensure_ascii=False), - encoding="utf-8", - ) - to_json(G, communities, str(out / "graph.json"), community_labels=labels) - labels_path.write_text(json.dumps({str(k): v for k, v in labels.items()}, ensure_ascii=False), encoding="utf-8") - # Membership signatures beside the labels so a later cluster-only can detect - # which communities changed and avoid reusing a stale label (see reuse above). - from graphify.cluster import community_member_sigs as _cms - (labels_path.parent / (labels_path.name + ".sig")).write_text( - json.dumps({str(k): v for k, v in _cms(communities).items()}), encoding="utf-8") - - # Mirror watch.py pattern: gate to_html so core outputs (graph.json + - # GRAPH_REPORT.md) always land. Honor --no-viz explicitly; otherwise - # fall back to ValueError handling so an oversized graph doesn't crash - # the CLI mid-write and leave a stale graph.html on disk. - html_target = out / "graph.html" - if no_viz: - if html_target.exists(): - html_target.unlink() - stages.mark("export"); stages.total() - print(f"Done - {len(communities)} communities. GRAPH_REPORT.md and graph.json updated (--no-viz; graph.html removed).") - else: - try: - # Over-cap fallback (#1019): force the community-aggregation - # path so an oversized graph still renders a usable graph.html. - _node_limit = 5000 if _over_cap else None - to_html(G, communities, str(html_target), community_labels=labels or None, - node_limit=_node_limit) - stages.mark("export"); stages.total() - print(f"Done - {len(communities)} communities. GRAPH_REPORT.md, graph.json and graph.html updated.") - except ValueError as viz_err: - if html_target.exists(): - html_target.unlink() - print(f"Skipped graph.html: {viz_err}") - stages.mark("export"); stages.total() - print(f"Done - {len(communities)} communities. GRAPH_REPORT.md and graph.json updated.") - - elif cmd == "update": - force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes") - no_cluster = False - args = sys.argv[2:] - watch_arg: str | None = None - for a in args: - if a == "--force": - force = True - continue - if a == "--no-cluster": - no_cluster = True - continue - if a.startswith("-"): - print(f"error: unknown update option: {a}", file=sys.stderr) - sys.exit(2) - if watch_arg is not None: - print("error: update accepts at most one path argument", file=sys.stderr) - sys.exit(2) - watch_arg = a - - if watch_arg is not None: - watch_path = Path(watch_arg) - else: - # Try to recover the scan root saved by the last full build - saved = Path(_GRAPHIFY_OUT) / ".graphify_root" - if saved.exists(): - watch_path = Path(saved.read_text(encoding="utf-8").strip()) - else: - watch_path = Path(".") - if not watch_path.exists(): - print(f"error: path not found: {watch_path}", file=sys.stderr) - sys.exit(1) - from graphify.watch import _rebuild_code - - print(f"Re-extracting code files in {watch_path} (no LLM needed)...") - # Interactive CLI: block on the per-repo lock rather than skip, so the - # user sees their explicit `graphify update` complete instead of - # exiting silently when a hook-driven rebuild happens to be running. - ok = _rebuild_code(watch_path, force=force, no_cluster=no_cluster, block_on_lock=True) - if ok: - print("Code graph updated. For doc/paper/image changes run /graphify --update in your AI assistant.") - if not ( - os.environ.get("GEMINI_API_KEY") - or os.environ.get("GOOGLE_API_KEY") - or os.environ.get("MOONSHOT_API_KEY") - or os.environ.get("DEEPSEEK_API_KEY") - or os.environ.get("GRAPHIFY_NO_TIPS") - ): - print("Tip: set GEMINI_API_KEY or GOOGLE_API_KEY to use Gemini for semantic extraction.") - else: - print( - "Nothing to update or rebuild failed — check output above.", - file=sys.stderr, - ) - sys.exit(1) - - elif cmd == "hook-check": - # Codex Desktop rejects hookSpecificOutput.additionalContext on PreToolUse. - # Keep this as a cross-platform no-op so installed hooks never break Bash - # tool calls. Graph guidance reaches the agent via AGENTS.md / skill instead. - sys.exit(0) - elif cmd == "check-update": - if len(sys.argv) < 3: - print("Usage: graphify check-update ", file=sys.stderr) - sys.exit(1) - from graphify.watch import check_update - - check_update(Path(sys.argv[2]).resolve()) - sys.exit(0) - elif cmd == "tree": - # Emit a D3 v7 collapsible-tree HTML view of graph.json: - # expand-all / collapse-all / reset-view buttons, multi-line - # wrapText labels with separately-coloured name + count, - # depth-based palette, click-to-toggle subtree, hover inspector - # showing top-K outbound edges per symbol. - from typing import Optional as _Opt - from graphify.tree_html import write_tree_html, DEFAULT_MAX_CHILDREN - graph_path = Path(_GRAPHIFY_OUT) / "graph.json" - output_path: "_Opt[Path]" = None - root: "_Opt[str]" = None - max_children = DEFAULT_MAX_CHILDREN - top_k_edges = 0 - project_label: "_Opt[str]" = None - args = sys.argv[2:] - i_arg = 0 - while i_arg < len(args): - a = args[i_arg] - if a == "--graph" and i_arg + 1 < len(args): - graph_path = Path(args[i_arg + 1]); i_arg += 2 - elif a == "--output" and i_arg + 1 < len(args): - output_path = Path(args[i_arg + 1]); i_arg += 2 - elif a == "--root" and i_arg + 1 < len(args): - root = args[i_arg + 1]; i_arg += 2 - elif a == "--max-children" and i_arg + 1 < len(args): - max_children = int(args[i_arg + 1]); i_arg += 2 - elif a == "--top-k-edges" and i_arg + 1 < len(args): - top_k_edges = int(args[i_arg + 1]); i_arg += 2 - elif a == "--label" and i_arg + 1 < len(args): - project_label = args[i_arg + 1]; i_arg += 2 - elif a in ("-h", "--help"): - print("Usage: graphify tree [--graph PATH] [--output HTML]") - print(" --graph PATH path to graph.json (default graphify-out/graph.json)") - print(" --output HTML output path (default graphify-out/GRAPH_TREE.html)") - print(" --root PATH filesystem root (default: longest common dir of all source_files)") - print(" --max-children N cap visible children per node (default 200)") - print(" --top-k-edges N pre-compute top-K outbound edges per symbol (default 12)") - print(" --label NAME project label shown in the page header") - return - else: - i_arg += 1 - if not graph_path.is_file(): - print(f"error: graph.json not found at {graph_path}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(graph_path) - if output_path is None: - output_path = graph_path.parent / "GRAPH_TREE.html" - out = write_tree_html( - graph_path=graph_path, output_path=output_path, - root=root, max_children=max_children, - top_k_edges=top_k_edges, project_label=project_label, - ) - size_kb = out.stat().st_size / 1024 - print(f"wrote {out} ({size_kb:.1f} KB)") - print(f"open with: xdg-open {out} (or file://{out.resolve()})") - sys.exit(0) - - elif cmd == "merge-driver": - # git merge driver for graph.json — takes (base, current, other) and writes - # the union of current+other nodes/edges back to current. Exits 1 on - # corrupt input so git surfaces the conflict instead of silently - # accepting a poisoned merge (see F-005). - # Usage: graphify merge-driver %O %A %B (set in .git/config merge driver) - if len(sys.argv) < 5: - print("Usage: graphify merge-driver ", file=sys.stderr) - sys.exit(1) - _base_path, _current_path, _other_path = sys.argv[2], sys.argv[3], sys.argv[4] - # Hard caps so a malicious or corrupted graph.json cannot exhaust memory - # at parse time. 50 MB / 100k nodes are well above any realistic graph - # (typical graphs are <5 MB / <50k nodes); anything larger should fail - # the merge so a human can investigate. - _MERGE_MAX_BYTES = 50 * 1024 * 1024 - _MERGE_MAX_NODES = 100_000 - import networkx as _nx - from networkx.readwrite import json_graph as _jg - def _load_graph(p: str): - path_obj = Path(p) - try: - size = path_obj.stat().st_size - except OSError as exc: - raise RuntimeError(f"cannot stat {p}: {exc}") from exc - if size > _MERGE_MAX_BYTES: - raise RuntimeError( - f"graph.json {p} is {size} bytes, exceeds {_MERGE_MAX_BYTES}-byte cap" - ) - data = json.loads(path_obj.read_text(encoding="utf-8")) - try: - return _jg.node_link_graph(data, edges="links"), data - except TypeError: - return _jg.node_link_graph(data), data - try: - G_cur, _ = _load_graph(_current_path) - G_oth, _ = _load_graph(_other_path) - except Exception as exc: - print(f"[graphify merge-driver] error loading graphs: {exc}", file=sys.stderr) - sys.exit(1) # surface the conflict so git doesn't accept a corrupt merge - merged = _nx.compose(G_cur, G_oth) - if merged.number_of_nodes() > _MERGE_MAX_NODES: - print( - f"[graphify merge-driver] merged graph has {merged.number_of_nodes()} nodes, " - f"exceeds {_MERGE_MAX_NODES}-node cap; aborting merge.", - file=sys.stderr, - ) - sys.exit(1) - try: - out_data = _jg.node_link_data(merged, edges="links") - except TypeError: - out_data = _jg.node_link_data(merged) - Path(_current_path).write_text(json.dumps(out_data, indent=2), encoding="utf-8") - sys.exit(0) - - elif cmd == "merge-graphs": - # graphify merge-graphs graph1.json graph2.json ... --out merged.json - args = sys.argv[2:] - graph_paths: list[Path] = [] - out_path = Path(_GRAPHIFY_OUT) / "merged-graph.json" - i = 0 - while i < len(args): - if args[i] == "--out" and i + 1 < len(args): - out_path = Path(args[i + 1]) - i += 2 - else: - graph_paths.append(Path(args[i])) - i += 1 - if len(graph_paths) < 2: - print( - "Usage: graphify merge-graphs [...] [--out merged.json]", - file=sys.stderr, - ) - sys.exit(1) - import networkx as _nx - from networkx.readwrite import json_graph as _jg - from graphify.build import prefix_graph_for_global as _prefix - graphs = [] - for gp in graph_paths: - if not gp.exists(): - print(f"error: not found: {gp}", file=sys.stderr) - sys.exit(1) - _enforce_graph_size_cap_or_exit(gp) - data = json.loads(gp.read_text(encoding="utf-8")) - # Normalize edges/links key before loading — graphify writes "links" - # via node_link_data but older runs may have used "edges" (#738). - if "links" not in data and "edges" in data: - data = dict(data, links=data["edges"]) - try: - G = _jg.node_link_graph(data, edges="links") - except TypeError: - G = _jg.node_link_graph(data) - graphs.append(G) - # nx.compose requires all graphs to be the same type. When input graphs - # come from different sources (e.g. an AST-only run vs a full LLM run) one - # may be a MultiGraph and another a Graph. Normalise everything to Graph - # (the graphify default) by converting MultiGraphs with nx.Graph(). - def _to_simple(g: "_nx.Graph") -> "_nx.Graph": - # nx.compose requires every graph to be the same type. Inputs may - # disagree on BOTH axes — directed vs undirected, and multi vs simple - # — because per-repo graph.json files are written by different extract - # paths at different times. Normalise everything to a plain undirected - # Graph (the merged cross-repo view is undirected anyway), which covers - # DiGraph / MultiGraph / MultiDiGraph. Without this a directed input - # crashed compose with "All graphs must be directed or undirected" (#1606). - if type(g) is not _nx.Graph: - return _nx.Graph(g) - return g - merged = _nx.Graph() - for G, gp in zip(graphs, graph_paths): - repo_tag = gp.parent.parent.name # graphify-out/../ → repo dir name - prefixed = _to_simple(_prefix(G, repo_tag)) - merged = _nx.compose(merged, prefixed) - try: - out_data = _jg.node_link_data(merged, edges="links") - except TypeError: - out_data = _jg.node_link_data(merged) - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(json.dumps(out_data, indent=2), encoding="utf-8") - print(f"Merged {len(graphs)} graphs -> {merged.number_of_nodes()} nodes, {merged.number_of_edges()} edges") - print(f"Written to: {out_path}") - - elif cmd == "clone": - if len(sys.argv) < 3: - print( - "Usage: graphify clone [--branch ] [--out ]", - file=sys.stderr, - ) - sys.exit(1) - url = sys.argv[2] - branch: str | None = None - out_dir: Path | None = None - args = sys.argv[3:] - i = 0 - while i < len(args): - if args[i] == "--branch" and i + 1 < len(args): - branch = args[i + 1] - i += 2 - elif args[i] == "--out" and i + 1 < len(args): - out_dir = Path(args[i + 1]) - i += 2 - else: - i += 1 - local_path = _clone_repo(url, branch=branch, out_dir=out_dir) - print(local_path) - - elif cmd == "export": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - if subcmd not in ("html", "callflow-html", "obsidian", "wiki", "svg", "graphml", "neo4j", "falkordb"): - print("Usage: graphify export ", file=sys.stderr) - print(" html [--graph PATH] [--labels PATH] [--node-limit N] [--no-viz]", file=sys.stderr) - print(" callflow-html [GRAPH|DIR] [--graph PATH] [--labels PATH] [--report PATH] [--sections PATH] [--output HTML]", file=sys.stderr) - print(" [--lang auto|zh-CN|en] [--max-sections N] [--diagram-scale N]", file=sys.stderr) - print(" obsidian [--graph PATH] [--labels PATH] [--dir PATH]", file=sys.stderr) - print(" wiki [--graph PATH] [--labels PATH]", file=sys.stderr) - print(" svg [--graph PATH] [--labels PATH]", file=sys.stderr) - print(" graphml [--graph PATH]", file=sys.stderr) - print(" neo4j [--graph PATH] [--push URI] [--user U] [--password P]", file=sys.stderr) - print(" (or set NEO4J_PASSWORD instead of --password to keep it off argv)", file=sys.stderr) - print(" falkordb [--graph PATH] [--push URI] [--user U] [--password P]", file=sys.stderr) - print(" (or set FALKORDB_PASSWORD instead of --password to keep it off argv)", file=sys.stderr) - sys.exit(1) - - # Parse shared args - args = sys.argv[3:] - graph_path = Path(_GRAPHIFY_OUT) / "graph.json" - graph_path_explicit = False - labels_path = Path(_GRAPHIFY_OUT) / ".graphify_labels.json" - labels_path_explicit = False - report_path = Path(_GRAPHIFY_OUT) / "GRAPH_REPORT.md" - report_path_explicit = False - sections_path: Path | None = None - callflow_output: Path | None = None - callflow_lang = "auto" - callflow_max_sections = 15 - callflow_diagram_scale = 1.0 - callflow_max_diagram_nodes = 18 - callflow_max_diagram_edges = 24 - analysis_path = Path(_GRAPHIFY_OUT) / ".graphify_analysis.json" - node_limit = 5000 - no_viz = False - obsidian_dir = Path(_GRAPHIFY_OUT) / "obsidian" - # Shared push-connection settings for the graph-database sinks (neo4j, - # falkordb), parsed from the generic --push/--user/--password flags below. - push_uri: str | None = None - push_user = "neo4j" # Neo4j default user; FalkorDB auth is optional and ignores it - # F-031: prefer an env var so the password never appears on argv (visible - # in `ps` output / shell history). The explicit --password flag still - # overrides it. Each sink reads its own var: FALKORDB_PASSWORD for falkordb, - # NEO4J_PASSWORD otherwise. - push_password: str | None = ( - os.environ.get("FALKORDB_PASSWORD") if subcmd == "falkordb" - else os.environ.get("NEO4J_PASSWORD") - ) or None - i = 0 - while i < len(args): - a = args[i] - if a == "--graph" and i + 1 < len(args): - graph_path = Path(args[i + 1]) - graph_path_explicit = True - i += 2 - elif a == "--labels" and i + 1 < len(args): - labels_path = Path(args[i + 1]) - labels_path_explicit = True - i += 2 - elif a == "--report" and i + 1 < len(args): - report_path = Path(args[i + 1]) - report_path_explicit = True - i += 2 - elif a == "--sections" and i + 1 < len(args): - sections_path = Path(args[i + 1]); i += 2 - elif a == "--output" and i + 1 < len(args): - callflow_output = Path(args[i + 1]).expanduser() - if not callflow_output.is_absolute(): - callflow_output = Path.cwd() / callflow_output - i += 2 - elif a == "--lang" and i + 1 < len(args): - callflow_lang = args[i + 1]; i += 2 - elif a == "--max-sections" and i + 1 < len(args): - callflow_max_sections = int(args[i + 1]); i += 2 - elif a == "--diagram-scale" and i + 1 < len(args): - callflow_diagram_scale = float(args[i + 1]); i += 2 - elif a == "--max-diagram-nodes" and i + 1 < len(args): - callflow_max_diagram_nodes = int(args[i + 1]); i += 2 - elif a == "--max-diagram-edges" and i + 1 < len(args): - callflow_max_diagram_edges = int(args[i + 1]); i += 2 - elif a in ("-h", "--help") and subcmd == "callflow-html": - print("Usage: graphify export callflow-html [GRAPH|DIR] [--graph PATH] [--labels PATH]") - print(" --report PATH path to GRAPH_REPORT.md") - print(" --sections PATH JSON section definitions") - print(" --output HTML output path (default graphify-out/-callflow.html)") - print(" --lang LANG auto, zh-CN, en, etc. (default auto)") - print(" --max-sections N maximum auto-derived sections (default 15)") - print(" --diagram-scale N Mermaid diagram scale (default 1.0)") - print(" --max-diagram-nodes N representative nodes per section (default 18)") - print(" --max-diagram-edges N representative edges per section (default 24)") - sys.exit(0) - elif a == "--node-limit" and i + 1 < len(args): - node_limit = int(args[i + 1]); i += 2 - elif a == "--no-viz": - no_viz = True; i += 1 - elif a == "--dir" and i + 1 < len(args): - obsidian_dir = Path(args[i + 1]); i += 2 - elif a == "--push" and i + 1 < len(args): - push_uri = args[i + 1]; i += 2 - elif a == "--user" and i + 1 < len(args): - push_user = args[i + 1]; i += 2 - elif a == "--password" and i + 1 < len(args): - push_password = args[i + 1]; i += 2 - elif subcmd == "callflow-html" and not a.startswith("-") and not graph_path_explicit: - candidate = Path(a) - if candidate.name == "graph.json" or candidate.suffix.lower() == ".json": - graph_path = candidate - elif (candidate / "graph.json").exists(): - graph_path = candidate / "graph.json" - else: - graph_path = candidate / _GRAPHIFY_OUT / "graph.json" - graph_path_explicit = True - i += 1 - else: - i += 1 - - graph_path = graph_path.expanduser() - if graph_path_explicit: - graph_out_dir = graph_path.parent - if not labels_path_explicit: - labels_path = graph_out_dir / ".graphify_labels.json" - if not report_path_explicit: - report_path = graph_out_dir / "GRAPH_REPORT.md" - labels_path = labels_path.expanduser() - report_path = report_path.expanduser() - - if not graph_path.exists(): - print(f"error: graph not found: {graph_path}. Run /graphify first.", file=sys.stderr) - sys.exit(1) - - if subcmd == "callflow-html": - from graphify.callflow_html import write_callflow_html as _write_callflow_html - out = _write_callflow_html( - graph=graph_path, - report=report_path, - labels=labels_path, - sections=sections_path, - output=callflow_output, - lang=callflow_lang, - max_sections=callflow_max_sections, - diagram_scale=callflow_diagram_scale, - max_diagram_nodes=callflow_max_diagram_nodes, - max_diagram_edges=callflow_max_diagram_edges, - verbose=True, - ) - print(f"callflow HTML written - open in any browser: {out}") - sys.exit(0) - - from networkx.readwrite import json_graph as _jg - from graphify.build import build_from_json as _bfj - from graphify.security import check_graph_file_size_cap as _check_cap - - # Solution 3 (#1019): for the HTML view, an oversized graph.json should - # not be a hard error. Detect the over-cap condition here and fall back - # to the community-aggregation view (node_limit=5000) below instead of - # exiting 1. All other subcommands keep the hard cap. - _over_cap = False - try: - _check_cap(graph_path) - except ValueError as _cap_err: - if subcmd == "html": - _over_cap = True - try: - _over_cap_bytes = graph_path.stat().st_size - except OSError: - _over_cap_bytes = -1 - print( - f"warning: graph.json exceeds cap ({_over_cap_bytes} bytes); " - f"falling back to community-aggregation view (node_limit=5000)", - file=sys.stderr, - ) - else: - print(f"error: {_cap_err}", file=sys.stderr) - sys.exit(1) - _raw = json.loads(graph_path.read_text(encoding="utf-8")) - if "links" not in _raw and "edges" in _raw: - _raw = dict(_raw, links=_raw["edges"]) - try: - G = _jg.node_link_graph(_raw, edges="links") - except TypeError: - G = _jg.node_link_graph(_raw) - - # Load optional analysis/labels - communities: dict[int, list[str]] = {} - if analysis_path.exists(): - _an = json.loads(analysis_path.read_text(encoding="utf-8")) - communities = {int(k): v for k, v in _an.get("communities", {}).items()} - cohesion: dict[int, float] = {int(k): v for k, v in _an.get("cohesion", {}).items()} - gods_data = _an.get("gods", []) - else: - cohesion = {} - gods_data = [] - - # Fallback: graph.json carries the per-node community as a node attribute - # (`to_json` writes it on every node). The analysis sidecar is the - # canonical source — but the post-commit / watch rebuild path doesn't - # regenerate it, and `extract` may have its temp files cleaned up. When - # that happens, `graphify export html` previously bailed with - # "Single community - aggregated view not useful." even though the - # per-node attribute had the right data all along. Reconstruct from - # the graph itself so downstream subcommands (html, obsidian, wiki, - # svg, graphml, neo4j) don't silently produce a degraded artifact. - if not communities: - reconstructed: dict[int, list[str]] = {} - for node_id, data in G.nodes(data=True): - cid_raw = data.get("community") - if cid_raw is None: - continue - try: - cid = int(cid_raw) - except (TypeError, ValueError): - continue - reconstructed.setdefault(cid, []).append(str(node_id)) - if reconstructed: - communities = reconstructed - - labels: dict[int, str] = {} - if labels_path.exists(): - labels = {int(k): v for k, v in json.loads(labels_path.read_text(encoding="utf-8")).items()} - - out_dir = graph_path.parent - - if subcmd == "html": - from graphify.export import to_html as _to_html - if no_viz: - html_target = out_dir / "graph.html" - if html_target.exists(): - html_target.unlink() - print("--no-viz: skipped graph.html") - else: - # Over-cap fallback (#1019): force the community-aggregation - # path so the oversized graph still renders a usable artifact. - _effective_node_limit = 5000 if _over_cap else node_limit - _to_html(G, communities, str(out_dir / "graph.html"), - community_labels=labels or None, node_limit=_effective_node_limit) - if G.number_of_nodes() <= _effective_node_limit: - print(f"graph.html written - open in any browser, no server needed") - if _over_cap: - sys.exit(0) - - elif subcmd == "obsidian": - from graphify.export import to_obsidian as _to_obsidian, to_canvas as _to_canvas - n = _to_obsidian(G, communities, str(obsidian_dir), - community_labels=labels or None, cohesion=cohesion or None) - print(f"Obsidian vault: {n} notes in {obsidian_dir}/") - _to_canvas(G, communities, str(obsidian_dir / "graph.canvas"), - community_labels=labels or None) - print(f"Canvas: {obsidian_dir}/graph.canvas") - print(f"Open {obsidian_dir}/ as a vault in Obsidian.") - - elif subcmd == "wiki": - from graphify.wiki import to_wiki as _to_wiki - from graphify.analyze import god_nodes as _god_nodes - if not communities: - print( - "error: .graphify_analysis.json is missing or empty — refusing to export wiki to prevent data loss.\n" - "Run `graphify extract .` (or `graphify cluster-only .`) to regenerate community data first.", - file=sys.stderr, - ) - sys.exit(1) - if not gods_data: - gods_data = _god_nodes(G) - n = _to_wiki(G, communities, str(out_dir / "wiki"), - community_labels=labels or None, cohesion=cohesion or None, - god_nodes_data=gods_data) - print(f"Wiki: {n} articles written to {out_dir}/wiki/") - print(f" {out_dir}/wiki/index.md -> agent entry point") - - elif subcmd == "svg": - from graphify.export import to_svg as _to_svg - _to_svg(G, communities, str(out_dir / "graph.svg"), - community_labels=labels or None) - print(f"graph.svg written - embeds in Obsidian, Notion, GitHub READMEs") - - elif subcmd == "graphml": - from graphify.export import to_graphml as _to_graphml - _to_graphml(G, communities, str(out_dir / "graph.graphml")) - print(f"graph.graphml written - open in Gephi, yEd, or any GraphML tool") - - elif subcmd == "neo4j": - if push_uri: - from graphify.export import push_to_neo4j as _push - if push_password is None: - print("error: --password required for --push", file=sys.stderr) - sys.exit(1) - result = _push(G, uri=push_uri, user=push_user, - password=push_password, communities=communities) - print(f"Pushed to Neo4j: {result['nodes']} nodes, {result['edges']} edges") - else: - from graphify.export import to_cypher as _to_cypher - _to_cypher(G, str(out_dir / "cypher.txt")) - print(f"cypher.txt written - import with: cypher-shell < {out_dir}/cypher.txt") - - elif subcmd == "falkordb": - if push_uri: - from graphify.export import push_to_falkordb as _push - result = _push(G, uri=push_uri, user=push_user, - password=push_password, communities=communities) - print(f"Pushed to FalkorDB: {result['nodes']} nodes, {result['edges']} edges") - else: - from graphify.export import to_cypher as _to_cypher - _to_cypher(G, str(out_dir / "cypher.txt")) - print(f"cypher.txt written ({out_dir}/cypher.txt) - statements are OpenCypher. " - f"FalkorDB's GRAPH.QUERY runs one statement at a time (no bulk script " - f"import), so load a graph with: graphify export falkordb --push " - f"falkordb://localhost:6379") - - elif cmd == "benchmark": - from graphify.benchmark import run_benchmark, print_benchmark - - graph_path = sys.argv[2] if len(sys.argv) > 2 else _default_graph_path() - _enforce_graph_size_cap_or_exit(Path(graph_path)) - # Try to load corpus_words from detect output - corpus_words = None - detect_path = Path(".graphify_detect.json") - if detect_path.exists(): - try: - detect_data = json.loads(detect_path.read_text(encoding="utf-8")) - corpus_words = detect_data.get("total_words") - except Exception: - pass - result = run_benchmark(graph_path, corpus_words=corpus_words) - print_benchmark(result) - - elif cmd == "global": - subcmd = sys.argv[2] if len(sys.argv) > 2 else "" - from graphify.global_graph import ( - global_add as _global_add, - global_remove as _global_remove, - global_list as _global_list, - global_path as _global_path, - ) - if subcmd == "add": - # graphify global add [--as ] - args = sys.argv[3:] - source = None - tag = None - i = 0 - while i < len(args): - if args[i] == "--as" and i + 1 < len(args): - tag = args[i + 1]; i += 2 - elif not source: - source = Path(args[i]); i += 1 - else: - i += 1 - if not source: - print("Usage: graphify global add [--as ]", file=sys.stderr) - sys.exit(1) - tag = tag or source.parent.parent.name - try: - result = _global_add(source, tag) - if result["skipped"]: - print(f"'{tag}' unchanged since last add - global graph not modified.") - else: - print(f"Added '{tag}' to global graph: +{result['nodes_added']} nodes, " - f"-{result['nodes_removed']} pruned. Global: {_global_path()}") - except Exception as exc: - print(f"error: {exc}", file=sys.stderr); sys.exit(1) - elif subcmd == "remove": - tag = sys.argv[3] if len(sys.argv) > 3 else "" - if not tag: - print("Usage: graphify global remove ", file=sys.stderr); sys.exit(1) - try: - removed = _global_remove(tag) - print(f"Removed '{tag}' from global graph ({removed} nodes pruned).") - except KeyError as exc: - print(f"error: {exc}", file=sys.stderr); sys.exit(1) - elif subcmd == "list": - repos = _global_list() - if not repos: - print("Global graph is empty. Use 'graphify global add' to add a project.") - else: - print(f"Global graph: {_global_path()}") - for tag, info in repos.items(): - print(f" {tag}: {info.get('node_count', '?')} nodes, added {info.get('added_at', '?')[:10]}") - elif subcmd == "path": - print(_global_path()) - else: - print("Usage: graphify global [add|remove|list|path]", file=sys.stderr); sys.exit(1) - - elif cmd == "extract": - # Headless full-pipeline extraction for CI / scripts (#698). - # Runs detect -> AST extraction on code -> semantic LLM extraction on - # docs/papers/images -> merge -> build -> cluster -> write outputs. - # Unlike the skill.md path (which runs through Claude Code subagents), - # this calls extract_corpus_parallel directly using whichever backend - # has an API key set. - if len(sys.argv) < 3: - print( - "Usage: graphify extract [--backend gemini|kimi|claude|openai|deepseek|ollama] " - "[--model M] [--mode deep] [--out DIR] [--google-workspace] [--no-cluster] " - "[--max-workers N] [--token-budget N] [--max-concurrency N] " - "[--api-timeout S] [--postgres DSN] [--cargo] [--timing]", - file=sys.stderr, - ) - sys.exit(1) - - has_path = True - if sys.argv[2].startswith("-"): - has_path = False - target = Path(".").resolve() - else: - target = Path(sys.argv[2]).resolve() - if not target.exists(): - print(f"error: path not found: {target}", file=sys.stderr) - sys.exit(1) - - backend: str | None = None - model: str | None = None - extract_mode: str | None = None - out_dir: Path | None = None - cli_postgres_dsn: str | None = None - cli_cargo: bool = False - no_cluster = False - dedup_llm = False - google_workspace = False - global_merge = False - global_repo_tag: str | None = None - # Performance/tuning knobs (issue #792). None means "use library default". - cli_max_workers: int | None = None - cli_token_budget: int | None = None - cli_max_concurrency: int | None = None - cli_api_timeout: float | None = None - # Clustering tuning knobs - cli_resolution: float = 1.0 - cli_exclude_hubs: float | None = None - cli_excludes: list[str] = [] - cli_timing: bool = False - - def _parse_int(name: str, raw: str) -> int: - try: - v = int(raw) - except ValueError: - print(f"error: {name} must be a positive integer (got {raw!r})", file=sys.stderr) - sys.exit(2) - if v <= 0: - print(f"error: {name} must be > 0 (got {v})", file=sys.stderr) - sys.exit(2) - return v - - def _parse_float(name: str, raw: str) -> float: - try: - v = float(raw) - except ValueError: - print(f"error: {name} must be a positive number (got {raw!r})", file=sys.stderr) - sys.exit(2) - if v <= 0: - print(f"error: {name} must be > 0 (got {v})", file=sys.stderr) - sys.exit(2) - return v - - args = sys.argv[3:] if has_path else sys.argv[2:] - i = 0 - while i < len(args): - a = args[i] - if a == "--backend" and i + 1 < len(args): - backend = args[i + 1]; i += 2 - elif a.startswith("--backend="): - backend = a.split("=", 1)[1]; i += 1 - elif a == "--model" and i + 1 < len(args): - model = args[i + 1]; i += 2 - elif a.startswith("--model="): - model = a.split("=", 1)[1]; i += 1 - elif a == "--mode" and i + 1 < len(args): - extract_mode = args[i + 1]; i += 2 - elif a.startswith("--mode="): - extract_mode = a.split("=", 1)[1]; i += 1 - elif a == "--out" and i + 1 < len(args): - out_dir = Path(args[i + 1]); i += 2 - elif a.startswith("--out="): - out_dir = Path(a.split("=", 1)[1]); i += 1 - elif a == "--no-cluster": - no_cluster = True; i += 1 - elif a == "--dedup-llm": - dedup_llm = True; i += 1 - elif a == "--google-workspace": - google_workspace = True; i += 1 - elif a == "--global": - global_merge = True; i += 1 - elif a == "--as" and i + 1 < len(args): - global_repo_tag = args[i + 1]; i += 2 - elif a == "--max-workers" and i + 1 < len(args): - cli_max_workers = _parse_int("--max-workers", args[i + 1]); i += 2 - elif a.startswith("--max-workers="): - cli_max_workers = _parse_int("--max-workers", a.split("=", 1)[1]); i += 1 - elif a == "--token-budget" and i + 1 < len(args): - cli_token_budget = _parse_int("--token-budget", args[i + 1]); i += 2 - elif a.startswith("--token-budget="): - cli_token_budget = _parse_int("--token-budget", a.split("=", 1)[1]); i += 1 - elif a == "--max-concurrency" and i + 1 < len(args): - cli_max_concurrency = _parse_int("--max-concurrency", args[i + 1]); i += 2 - elif a.startswith("--max-concurrency="): - cli_max_concurrency = _parse_int("--max-concurrency", a.split("=", 1)[1]); i += 1 - elif a == "--api-timeout" and i + 1 < len(args): - cli_api_timeout = _parse_float("--api-timeout", args[i + 1]); i += 2 - elif a.startswith("--api-timeout="): - cli_api_timeout = _parse_float("--api-timeout", a.split("=", 1)[1]); i += 1 - elif a == "--resolution" and i + 1 < len(args): - cli_resolution = _parse_float("--resolution", args[i + 1]); i += 2 - elif a.startswith("--resolution="): - cli_resolution = _parse_float("--resolution", a.split("=", 1)[1]); i += 1 - elif a == "--exclude-hubs" and i + 1 < len(args): - cli_exclude_hubs = float(args[i + 1]); i += 2 - elif a.startswith("--exclude-hubs="): - cli_exclude_hubs = float(a.split("=", 1)[1]); i += 1 - elif a == "--exclude" and i + 1 < len(args): - cli_excludes.append(args[i + 1]); i += 2 - elif a.startswith("--exclude="): - cli_excludes.append(a.split("=", 1)[1]); i += 1 - elif a == "--postgres" and i + 1 < len(args): - cli_postgres_dsn = args[i + 1]; i += 2 - elif a.startswith("--postgres="): - cli_postgres_dsn = a.split("=", 1)[1]; i += 1 - elif a == "--cargo": - cli_cargo = True - i += 1 - elif a == "--timing": - cli_timing = True; i += 1 - else: - i += 1 - - if not has_path and cli_postgres_dsn is None: - print("error: must specify a path to scan or a --postgres DSN", file=sys.stderr) - sys.exit(1) - - _VALID_MODES = {"deep"} - if extract_mode is not None and extract_mode not in _VALID_MODES: - print( - f"error: unknown --mode '{extract_mode}'. " - f"Available: {', '.join(sorted(_VALID_MODES))}", - file=sys.stderr, - ) - sys.exit(2) - deep_mode = extract_mode == "deep" - if deep_mode: - print("[graphify extract] deep mode enabled: richer semantic extraction") - - # CLI flag wins over env var. Setting GRAPHIFY_API_TIMEOUT here so - # _call_openai_compat picks it up without needing a new kwarg path. - if cli_api_timeout is not None: - os.environ["GRAPHIFY_API_TIMEOUT"] = str(cli_api_timeout) - if cli_max_workers is not None: - os.environ["GRAPHIFY_MAX_WORKERS"] = str(cli_max_workers) - - # Resolve output dir. The user-facing contract is "/graphify-out/" - # so a fresh checkout writes graphify-out/ at the project root, matching - # the skill.md pipeline. - out_root = (out_dir.resolve() if out_dir else target) - graphify_out = out_root / _GRAPHIFY_OUT - graphify_out.mkdir(parents=True, exist_ok=True) - - stages = _StageTimer(cli_timing) - - from graphify.detect import ( - detect as _detect, - detect_incremental as _detect_incremental, - save_manifest as _save_manifest, - ) - manifest_path = graphify_out / "manifest.json" - existing_graph_path = graphify_out / "graph.json" - incremental_mode = manifest_path.exists() and existing_graph_path.exists() if has_path else False - - if not has_path: - code_files = [] - doc_files = [] - paper_files = [] - image_files = [] - deleted_files = [] - unchanged_total = 0 - files_by_type = {} - elif incremental_mode: - print(f"[graphify extract] incremental scan of {target}") - detection = _detect_incremental( - target, - manifest_path=str(manifest_path), - google_workspace=google_workspace or None, - extra_excludes=cli_excludes or None, - ) - files_by_type = detection.get("files", {}) - new_by_type = detection.get("new_files", {}) - code_files = [Path(p) for p in new_by_type.get("code", [])] - doc_files = [Path(p) for p in new_by_type.get("document", [])] - paper_files = [Path(p) for p in new_by_type.get("paper", [])] - image_files = [Path(p) for p in new_by_type.get("image", [])] - deleted_files = list(detection.get("deleted_files", [])) - unchanged_total = sum(len(v) for v in detection.get("unchanged_files", {}).values()) - else: - print(f"[graphify extract] scanning {target}") - detection = _detect(target, google_workspace=google_workspace or None, extra_excludes=cli_excludes or None) - files_by_type = detection.get("files", {}) - code_files = [Path(p) for p in files_by_type.get("code", [])] - doc_files = [Path(p) for p in files_by_type.get("document", [])] - paper_files = [Path(p) for p in files_by_type.get("paper", [])] - image_files = [Path(p) for p in files_by_type.get("image", [])] - deleted_files = [] - unchanged_total = 0 - - semantic_files = doc_files + paper_files + image_files - if incremental_mode: - print( - f"[graphify extract] {len(code_files)} code, {len(doc_files)} docs, " - f"{len(paper_files)} papers, {len(image_files)} images changed; " - f"{unchanged_total} unchanged; {len(deleted_files)} deleted" - ) - else: - print( - f"[graphify extract] found {len(code_files)} code, " - f"{len(doc_files)} docs, {len(paper_files)} papers, " - f"{len(image_files)} images" - ) - stages.mark("detect") - - # Resolve the LLM backend only now that we know whether the corpus - # needs one. A code-only corpus is pure local AST and must not require - # an API key; the key is enforced below only when there's LLM work. - from graphify.llm import ( - BACKENDS as _BACKENDS, - detect_backend as _detect_backend, - estimate_cost as _estimate_cost, - extract_corpus_parallel as _extract_corpus_parallel, - _format_backend_env_keys, - _get_backend_api_key, - ) - needs_llm = bool(semantic_files) or dedup_llm - if backend is None and needs_llm: - backend = _detect_backend() - if backend is not None and backend not in _BACKENDS: - print( - f"error: unknown backend '{backend}'. " - f"Available: {', '.join(sorted(_BACKENDS))}", - file=sys.stderr, - ) - sys.exit(1) - if needs_llm: - if backend is None: - reasons = [] - if semantic_files: - reasons.append( - f"{len(semantic_files)} doc/paper/image file(s) need semantic extraction" - ) - if dedup_llm: - reasons.append("--dedup-llm was passed") - print( - "error: no LLM API key found (" + "; ".join(reasons) + "). " - "Set GEMINI_API_KEY or GOOGLE_API_KEY (gemini), MOONSHOT_API_KEY " - "(kimi), ANTHROPIC_API_KEY (claude), OPENAI_API_KEY (openai), " - "DEEPSEEK_API_KEY (deepseek), or pass --backend. A code-only " - "corpus needs no key.", - file=sys.stderr, - ) - sys.exit(1) - if backend == "ollama": - from graphify.llm import _validate_ollama_base_url - _oll_url = os.environ.get("OLLAMA_BASE_URL", _BACKENDS["ollama"].get("base_url", "")) - try: - _validate_ollama_base_url(_oll_url, warn=False) - except ValueError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(2) - if not _get_backend_api_key(backend): - allow_no_key = False - if backend == "ollama": - from urllib.parse import urlparse - ollama_url = os.environ.get( - "OLLAMA_BASE_URL", - _BACKENDS["ollama"].get("base_url", ""), - ) - try: - host = (urlparse(ollama_url).hostname or "").lower() - except Exception: - host = "" - allow_no_key = ( - host in ("localhost", "127.0.0.1", "::1") - or host.startswith("127.") - ) - elif backend == "bedrock": - allow_no_key = bool( - os.environ.get("AWS_PROFILE") - or os.environ.get("AWS_REGION") - or os.environ.get("AWS_DEFAULT_REGION") - or os.environ.get("AWS_ACCESS_KEY_ID") - ) - elif backend == "claude-cli": - import shutil as _shutil - allow_no_key = _shutil.which("claude") is not None - if not allow_no_key: - print( - "error: backend 'claude-cli' requires the `claude` CLI on $PATH " - "(install Claude Code and run `claude` once to authenticate).", - file=sys.stderr, - ) - sys.exit(1) - if not allow_no_key: - print( - f"error: backend '{backend}' requires {_format_backend_env_keys(backend)} to be set.", - file=sys.stderr, - ) - sys.exit(1) - - # AST extraction on code files. Empty code list (docs-only corpus) is - # the issue #698 case — skip cleanly instead of crashing inside extract(). - ast_result: dict = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} - if code_files: - from graphify.extract import extract as _ast_extract - # Anchor the cache at the output root, not the scanned project: - # with --out, a /graphify-out/cache/ would leak a - # graphify-out/ dir into a project that asked for external output. - ast_kwargs: dict = {"cache_root": out_root} - if cli_max_workers is not None: - ast_kwargs["max_workers"] = cli_max_workers - print(f"[graphify extract] AST extraction on {len(code_files)} code files...") - try: - ast_result = _ast_extract(code_files, **ast_kwargs) - except Exception as exc: - print(f"[graphify extract] AST extraction failed: {exc}", file=sys.stderr) - ast_result = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} - stages.mark("AST extract") - - # Semantic extraction on docs/papers/images. Check cache first. - from graphify.cache import ( - check_semantic_cache as _check_semantic_cache, - prune_semantic_cache as _prune_semantic_cache, - save_semantic_cache as _save_semantic_cache, - ) - sem_result: dict = { - "nodes": [], "edges": [], "hyperedges": [], - "input_tokens": 0, "output_tokens": 0, - } - sem_cache_hits = 0 - sem_cache_misses = 0 - if semantic_files: - sem_paths_str = [str(p) for p in semantic_files] - cached_nodes, cached_edges, cached_hyperedges, uncached_paths = ( - _check_semantic_cache(sem_paths_str, root=out_root) - ) - sem_cache_hits = len(semantic_files) - len(uncached_paths) - sem_cache_misses = len(uncached_paths) - sem_result["nodes"].extend(cached_nodes) - sem_result["edges"].extend(cached_edges) - sem_result["hyperedges"].extend(cached_hyperedges) - if sem_cache_hits: - print(f"[graphify extract] semantic cache: {sem_cache_hits} hit / {sem_cache_misses} miss") - - if uncached_paths: - print(f"[graphify extract] semantic extraction on {len(uncached_paths)} files via {backend}...") - corpus_kwargs: dict = { - "backend": backend, - "model": model, - "root": target, - } - if deep_mode: - corpus_kwargs["deep_mode"] = True - if cli_token_budget is not None: - corpus_kwargs["token_budget"] = cli_token_budget - if cli_max_concurrency is not None: - corpus_kwargs["max_concurrency"] = cli_max_concurrency - - # Minimal progress callback so the CLI is no longer silent - # during long local-inference runs (issue #792 addendum). - # Also track per-chunk success so we can fail loudly when - # every chunk errors (e.g. missing backend SDK package). - _chunk_stats = {"total": 0, "succeeded": 0} - def _progress(idx: int, total: int, _result: dict) -> None: - _chunk_stats["total"] = total - _chunk_stats["succeeded"] += 1 - print( - f"[graphify extract] chunk {idx + 1}/{total} done", - flush=True, - ) - corpus_kwargs["on_chunk_done"] = _progress - - try: - fresh = _extract_corpus_parallel( - [Path(p) for p in uncached_paths], - **corpus_kwargs, - ) - except ImportError as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - except Exception as exc: - print( - f"[graphify extract] semantic extraction failed: {exc}", - file=sys.stderr, - ) - fresh = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} - - # on_chunk_done only fires after a chunk succeeds. If fresh - # semantic extraction was requested and no chunks completed, - # fail instead of writing an AST-only graph with exit 0. - if uncached_paths and _chunk_stats["succeeded"] == 0: - print( - f"[graphify extract] error: all semantic chunks failed " - f"for backend '{backend}' ({len(uncached_paths)} uncached files) - " - f"see per-chunk errors above. If you see 'requires the X package', " - f"run `pip install X` and retry.", - file=sys.stderr, - ) - sys.exit(1) - try: - _save_semantic_cache( - fresh.get("nodes", []), - fresh.get("edges", []), - fresh.get("hyperedges", []), - root=out_root, - ) - except Exception as exc: - print(f"[graphify extract] warning: could not write semantic cache: {exc}", file=sys.stderr) - sem_result["nodes"].extend(fresh.get("nodes", [])) - sem_result["edges"].extend(fresh.get("edges", [])) - sem_result["hyperedges"].extend(fresh.get("hyperedges", [])) - sem_result["input_tokens"] += fresh.get("input_tokens", 0) - sem_result["output_tokens"] += fresh.get("output_tokens", 0) - - # Prune orphaned semantic cache entries. The semantic cache is - # content-hash-keyed and unversioned, so it is never swept by the AST - # version-cleanup: every content change or file deletion leaves a - # permanent orphan that accumulates unbounded (#1527). Sweep it against - # the FULL live document set (``files_by_type`` — present in both the - # incremental and full branches), NOT the incremental ``semantic_files`` - # changed-subset, which would delete every unchanged doc's valid entry. - # Best-effort: a prune failure must never break extraction. - try: - from graphify.cache import file_hash as _file_hash - _live_hashes: set[str] = set() - for _kind in ("document", "paper", "image"): - for _fp in files_by_type.get(_kind, []): - _abs = Path(_fp) - if not _abs.is_absolute(): - _abs = Path(out_root) / _abs - if not _abs.is_file(): - continue # deleted/missing — leave out so its entry is pruned - try: - _live_hashes.add(_file_hash(_abs, out_root)) - except OSError: - pass - _prune_semantic_cache(out_root, _live_hashes) - except Exception as exc: - print(f"[graphify extract] warning: could not prune semantic cache: {exc}", file=sys.stderr) - stages.mark("semantic extract") - - pg_result: dict = {"nodes": [], "edges": []} - if cli_postgres_dsn is not None: - from graphify.pg_introspect import introspect_postgres - print(f"[graphify extract] introspecting PostgreSQL schema...") - try: - pg_result = introspect_postgres(cli_postgres_dsn) - except (ConnectionError, ImportError) as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - print(f"[graphify extract] PostgreSQL: {len(pg_result['nodes'])} nodes, " - f"{len(pg_result['edges'])} edges") - - cargo_result: dict = {"nodes": [], "edges": []} - if cli_cargo: - from graphify.cargo_introspect import introspect_cargo - print("[graphify extract] introspecting Cargo workspace...") - try: - cargo_result = introspect_cargo(target) - except (ConnectionError, ImportError, OSError) as exc: - print(f"error: {exc}", file=sys.stderr) - sys.exit(1) - print(f"[graphify extract] Cargo: {len(cargo_result['nodes'])} nodes, " - f"{len(cargo_result['edges'])} edges") - - # Merge AST + semantic + pg_result + cargo_result. Order matters for deduplication: passing AST - # first means semantic node attributes win on collision (richer labels - # for symbols also referenced in docs). Hyperedges only come from the - # semantic side. - merged: dict = { - "nodes": list(ast_result.get("nodes", [])) + list(sem_result.get("nodes", [])) + list(pg_result.get("nodes", [])) + list(cargo_result.get("nodes", [])), - "edges": list(ast_result.get("edges", [])) + list(sem_result.get("edges", [])) + list(pg_result.get("edges", [])) + list(cargo_result.get("edges", [])), - "hyperedges": list(sem_result.get("hyperedges", [])), - "input_tokens": ast_result.get("input_tokens", 0) + sem_result.get("input_tokens", 0), - "output_tokens": ast_result.get("output_tokens", 0) + sem_result.get("output_tokens", 0), - } - - graph_json_path = graphify_out / "graph.json" - analysis_path = graphify_out / ".graphify_analysis.json" - - # Build a manifest-safe files dict: only stamp semantic_hash for files - # that actually produced output (cache hit or fresh extraction). Files - # whose chunk failed have no source_file entry in sem_result — leaving - # their semantic_hash empty so detect_incremental re-queues them (#933). - _sem_extracted: set[str] = { - n.get("source_file", "") for n in sem_result.get("nodes", []) - } | { - e.get("source_file", "") for e in sem_result.get("edges", []) - } - _sem_extracted.discard("") - _sem_types = {"document", "paper", "image"} - _manifest_files = { - ftype: [f for f in flist if ftype not in _sem_types or f in _sem_extracted] - for ftype, flist in files_by_type.items() - } - - if no_cluster: - # --no-cluster: dump the raw merged extraction as graph.json. - # No NetworkX, no community detection, no analysis sidecar. - # Dedupe nodes (by id) and parallel edges so the raw output matches the - # clustered path (whose DiGraph collapses both) and stays deterministic - # across modes (#1317; node dedup also collapses shared Swift module - # anchors emitted per importing file, #1327). - from graphify.build import dedupe_edges as _dedupe_edges, dedupe_nodes as _dedupe_nodes - from graphify.export import backup_if_protected as _backup - if ( - incremental_mode - and not code_files - and not semantic_files - and not deleted_files - and not pg_result.get("nodes") - and not pg_result.get("edges") - and not cargo_result.get("nodes") - and not cargo_result.get("edges") - ): - print( - "[graphify extract] no incremental changes detected " - "(--no-cluster); outputs left untouched." - ) - try: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) - except Exception as exc: - print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) - stages.total() - sys.exit(0) - - merged["nodes"] = _dedupe_nodes(merged["nodes"]) - merged["edges"] = _dedupe_edges(merged["edges"]) - # Backfill source_file from endpoint nodes — this raw path bypasses - # build_from_json's backfill, and semantic edges sometimes omit it (#1279). - _node_sf = {n.get("id"): n.get("source_file") for n in merged["nodes"]} - for _e in merged["edges"]: - if not _e.get("source_file"): - _e["source_file"] = ( - _node_sf.get(_e.get("source")) or _node_sf.get(_e.get("target")) or "" - ) - _backup(graphify_out) - graph_json_path.write_text( - json.dumps(merged, indent=2), encoding="utf-8" - ) - stages.mark("write") - cost = _estimate_cost( - backend, merged["input_tokens"], merged["output_tokens"] - ) - print( - f"[graphify extract] wrote {graph_json_path} — " - f"{len(merged['nodes'])} nodes, {len(merged['edges'])} edges " - f"(no clustering)" - ) - if merged["input_tokens"] or merged["output_tokens"]: - print( - f"[graphify extract] tokens: " - f"{merged['input_tokens']:,} in / " - f"{merged['output_tokens']:,} out, " - f"est. cost: ${cost:.4f}" - ) - try: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) - except Exception as exc: - print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) - if global_merge: - from graphify.global_graph import global_add as _global_add - _tag = global_repo_tag or target.name - try: - result = _global_add(graphify_out / "graph.json", _tag) - if result["skipped"]: - print(f"[graphify global] '{_tag}' unchanged since last add - skipped.") - else: - print(f"[graphify global] '{_tag}' merged into global graph " - f"(+{result['nodes_added']} nodes, -{result['nodes_removed']} pruned).") - except Exception as exc: - print(f"[graphify global] warning: failed to merge into global graph: {exc}", file=sys.stderr) - stages.total() - sys.exit(0) - - # Build graph + cluster + score + write. - from graphify.build import ( - build as _build, - build_from_json as _build_from_json, - build_merge as _build_merge, - ) - from graphify.cluster import cluster as _cluster, score_all as _score_all - from graphify.export import to_json as _to_json - from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising - dedup_backend = backend if dedup_llm else None - if incremental_mode: - G = _build_merge( - [merged], - graph_path=existing_graph_path, - prune_sources=deleted_files or None, - dedup=True, - dedup_llm_backend=dedup_backend, - root=target, - ) - else: - G = _build([merged], dedup=True, dedup_llm_backend=dedup_backend, root=target) - stages.mark("build") - if G.number_of_nodes() == 0: - print( - "[graphify extract] graph is empty — extraction produced no nodes. " - "Possible causes: all files skipped, binary-only corpus, or LLM " - "returned no edges.", - file=sys.stderr, - ) - sys.exit(1) - - communities = _cluster(G, resolution=cli_resolution, exclude_hubs_percentile=cli_exclude_hubs) - stages.mark("cluster") - cohesion = _score_all(G, communities) - try: - gods = _god_nodes(G) - except Exception: - gods = [] - try: - surprises = _surprising(G, communities) - except Exception: - surprises = [] - stages.mark("analyze") - - from graphify.export import backup_if_protected as _backup - _backup(graphify_out) - _to_json(G, communities, str(graph_json_path), force=True) - stages.mark("export") - if merged.get("output_tokens", 0) > 0: - (graphify_out / ".graphify_semantic_marker").write_text( - json.dumps({"output_tokens": merged["output_tokens"]}), encoding="utf-8" - ) - if global_merge: - from graphify.global_graph import global_add as _global_add - _tag = global_repo_tag or target.name - try: - result = _global_add(graphify_out / "graph.json", _tag) - if result["skipped"]: - print(f"[graphify global] '{_tag}' unchanged since last add - skipped.") - else: - print(f"[graphify global] '{_tag}' merged into global graph " - f"(+{result['nodes_added']} nodes, -{result['nodes_removed']} pruned).") - except Exception as exc: - print(f"[graphify global] warning: failed to merge into global graph: {exc}", file=sys.stderr) - analysis = { - "communities": {str(k): v for k, v in communities.items()}, - "cohesion": {str(k): v for k, v in cohesion.items()}, - "gods": gods, - "surprises": surprises, - "tokens": { - "input": merged["input_tokens"], - "output": merged["output_tokens"], - }, - } - analysis_path.write_text(json.dumps(analysis, indent=2), encoding="utf-8") - try: - _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target) - except Exception as exc: - print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) - - cost = _estimate_cost(backend, merged["input_tokens"], merged["output_tokens"]) - print( - f"[graphify extract] wrote {graph_json_path}: " - f"{G.number_of_nodes()} nodes, {G.number_of_edges()} edges, " - f"{len(communities)} communities" - ) - print(f"[graphify extract] wrote {analysis_path}") - if incremental_mode: - print( - f"[graphify extract] incremental summary: " - f"{sem_cache_hits + unchanged_total} files cached/unchanged, " - f"{len(code_files) + sem_cache_misses} re-extracted, " - f"{len(deleted_files)} deleted" - ) - elif sem_cache_hits: - print(f"[graphify extract] semantic cache: {sem_cache_hits} cached, {sem_cache_misses} re-extracted") - if merged["input_tokens"] or merged["output_tokens"]: - print( - f"[graphify extract] tokens: " - f"{merged['input_tokens']:,} in / " - f"{merged['output_tokens']:,} out, " - f"est. cost (~{backend}): ${cost:.4f}" - ) - # extract intentionally stops at graph.json + analysis; the report and - # community labels are produced by `cluster-only` (or an agent's Step 5). - # Point standalone users at it so communities get named (#1097). - print( - "[graphify extract] next: run " - f"`graphify cluster-only {graphify_out.parent}` " - "to generate GRAPH_REPORT.md and name communities" - ) - stages.total() - - elif cmd == "cache-check": - # graphify cache-check [--root ] - # Reads file paths (one per line) from , checks semantic cache. - # Writes: - # graphify-out/.graphify_cached.json — already-cached nodes/edges/hyperedges - # graphify-out/.graphify_uncached.txt — paths that need extraction - # Stdout: "Cache: N hit, M miss" - from graphify.cache import check_semantic_cache - if len(sys.argv) < 3: - print("Usage: graphify cache-check [--root ]", file=sys.stderr) - sys.exit(1) - files_from = Path(sys.argv[2]) - root = Path(".") - i = 3 - while i < len(sys.argv): - if sys.argv[i] == "--root" and i + 1 < len(sys.argv): - root = Path(sys.argv[i + 1]) - i += 2 - else: - i += 1 - files = [f for f in files_from.read_text(encoding="utf-8").splitlines() if f.strip()] - cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(files, root) - out = root / _GRAPHIFY_OUT - out.mkdir(parents=True, exist_ok=True) - if cached_nodes or cached_edges or cached_hyperedges: - (out / ".graphify_cached.json").write_text( - json.dumps({"nodes": cached_nodes, "edges": cached_edges, "hyperedges": cached_hyperedges}, - ensure_ascii=False), - encoding="utf-8", - ) - (out / ".graphify_uncached.txt").write_text("\n".join(uncached), encoding="utf-8") - print(f"Cache: {len(files) - len(uncached)} hit, {len(uncached)} miss") - - elif cmd == "merge-chunks": - # graphify merge-chunks --out - # Concatenates .graphify_chunk_*.json files written by semantic subagents. - # Deduplicates nodes by id (first writer wins). Sums token counts. - import glob as _glob - if len(sys.argv) < 3: - print("Usage: graphify merge-chunks --out ", file=sys.stderr) - sys.exit(1) - out_path: Path | None = None - chunk_args: list[str] = [] - i = 2 - while i < len(sys.argv): - if sys.argv[i] == "--out" and i + 1 < len(sys.argv): - out_path = Path(sys.argv[i + 1]) - i += 2 - else: - chunk_args.append(sys.argv[i]) - i += 1 - if not out_path: - print("error: --out required", file=sys.stderr) - sys.exit(1) - chunk_files: list[str] = [] - for arg in chunk_args: - expanded = _glob.glob(arg) - chunk_files.extend(sorted(expanded) if expanded else [arg]) - merged: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} - seen_ids: set[str] = set() - for cf in chunk_files: - try: - chunk = json.loads(Path(cf).read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError) as exc: - print(f"[graphify merge-chunks] warning: skipping {cf}: {exc}", file=sys.stderr) - continue - for n in chunk.get("nodes", []): - if n.get("id") not in seen_ids: - seen_ids.add(n["id"]) - merged["nodes"].append(n) - merged["edges"].extend(chunk.get("edges", [])) - merged["hyperedges"].extend(chunk.get("hyperedges", [])) - merged["input_tokens"] += chunk.get("input_tokens", 0) - merged["output_tokens"] += chunk.get("output_tokens", 0) - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(json.dumps(merged, ensure_ascii=False), encoding="utf-8") - print( - f"Merged {len(chunk_files)} chunks: {len(merged['nodes'])} nodes, {len(merged['edges'])} edges, " - f"{merged['input_tokens']:,} in / {merged['output_tokens']:,} out tokens" - ) - - elif cmd == "merge-semantic": - # graphify merge-semantic --cached --new --out - # Merges cached semantic results with freshly-extracted chunk results. - # Deduplicates nodes by id (cached entries take priority over new ones). - if len(sys.argv) < 3: - print("Usage: graphify merge-semantic --cached --new --out ", file=sys.stderr) - sys.exit(1) - cached_path: Path | None = None - new_path: Path | None = None - out_path2: Path | None = None - i = 2 - while i < len(sys.argv): - if sys.argv[i] == "--cached" and i + 1 < len(sys.argv): - cached_path = Path(sys.argv[i + 1]); i += 2 - elif sys.argv[i] == "--new" and i + 1 < len(sys.argv): - new_path = Path(sys.argv[i + 1]); i += 2 - elif sys.argv[i] == "--out" and i + 1 < len(sys.argv): - out_path2 = Path(sys.argv[i + 1]); i += 2 - else: - i += 1 - if not out_path2: - print("error: --out required", file=sys.stderr) - sys.exit(1) - empty: dict = {"nodes": [], "edges": [], "hyperedges": []} - cached_data = json.loads(cached_path.read_text(encoding="utf-8")) if cached_path and cached_path.exists() else empty - new_data = json.loads(new_path.read_text(encoding="utf-8")) if new_path and new_path.exists() else empty - seen_ids2: set[str] = set() - all_nodes: list[dict] = [] - for n in cached_data.get("nodes", []) + new_data.get("nodes", []): - if n.get("id") not in seen_ids2: - seen_ids2.add(n["id"]) - all_nodes.append(n) - merged2 = { - "nodes": all_nodes, - "edges": cached_data.get("edges", []) + new_data.get("edges", []), - "hyperedges": cached_data.get("hyperedges", []) + new_data.get("hyperedges", []), - } - out_path2.parent.mkdir(parents=True, exist_ok=True) - out_path2.write_text(json.dumps(merged2, ensure_ascii=False), encoding="utf-8") - print(f"Merged: {len(merged2['nodes'])} nodes, {len(merged2['edges'])} edges") - - elif Path(cmd).exists() or cmd in (".", "..") or cmd.startswith(("./", "../", "/", "~")): - # User ran `graphify ` directly — treat as `graphify extract `. - # Common when following the PowerShell note in README (`graphify .`) or - # copy-pasting skill invocations without the leading slash. - sys.argv.insert(2, sys.argv[1]) - sys.argv[1] = "extract" - main() - else: - print(f"error: unknown command '{cmd}'", file=sys.stderr) - print("Run 'graphify --help' for usage.", file=sys.stderr) - sys.exit(1) + if dispatch_install_cli(cmd): + return + dispatch_command(cmd) if __name__ == "__main__": diff --git a/graphify/affected.py b/graphify/affected.py index deacc39c8..4fc7b8fbf 100644 --- a/graphify/affected.py +++ b/graphify/affected.py @@ -30,6 +30,11 @@ class AffectedHit: node_id: str depth: int via_relation: str + # The traversed edge's location — the actual call/import/reference SITE in + # this node's file, not the node's own definition line (#BUG1). Defaults keep + # existing constructors/tests working; None falls back to the node's def line. + via_file: "str | None" = None + via_location: "str | None" = None def _node_label(graph: nx.Graph, node_id: str) -> str: @@ -190,7 +195,15 @@ def affected_nodes( if source in seen: continue seen.add(source) - hit = AffectedHit(source, current_depth + 1, relation) + # Carry the matched edge's location (taken from the SAME edge dict + # whose relation passed the filter, so relation and location stay + # consistent) — that is the call/import/reference site in `source`'s + # own file, which is where the user should click (#BUG1). + hit = AffectedHit( + source, current_depth + 1, relation, + via_file=str(data.get("source_file") or "") or None, + via_location=str(data.get("source_location") or "") or None, + ) hits.append(hit) queue.append((source, current_depth + 1)) @@ -221,8 +234,14 @@ def format_affected( for hit in hits: data = graph.nodes[hit.node_id] + if hit.via_location: + # The relation SITE in this node's file (call/import/reference line), + # labeled by [via_relation] so it's never mistaken for a def line. + location = f"{hit.via_file or data.get('source_file') or '-'}:{hit.via_location}" + else: + location = _format_location(data) # honest fallback: the node's own def line lines.append( - f"- {_node_label(graph, hit.node_id)} [{hit.via_relation}] {_format_location(data)}" + f"- {_node_label(graph, hit.node_id)} [{hit.via_relation}] {location}" ) return "\n".join(lines) diff --git a/graphify/analyze.py b/graphify/analyze.py index 0d4bbe4c9..0707e2be7 100644 --- a/graphify/analyze.py +++ b/graphify/analyze.py @@ -18,17 +18,25 @@ "Callable", "Type", "ClassVar", "Final", "Literal", "Protocol", "Counter", "defaultdict", "OrderedDict", "datetime", "Enum", "os", "sys", "re", "json", "io", "abc", "typing", + # Swift / Foundation / SwiftUI framework symbols and module imports that + # otherwise dominate god-node rankings on Swift codebases (#2147) + "Foundation", "SwiftUI", "UIKit", "AppKit", "Combine", + "String", "Int", "Double", "Float", "Bool", "Data", "URL", "Date", "UUID", + "Sendable", "Codable", "Decodable", "Encodable", "Equatable", "Hashable", + "Identifiable", "Comparable", "AnyObject", "Error", "LocalizedError", + "NSObject", "NSString", "NSError", "NSLock", + "View", "Color", "Font", "DispatchQueue", }) # Language families — extensions sharing a runtime can legitimately call each other _LANG_FAMILY: dict[str, str] = { **{e: "python" for e in (".py", ".pyw")}, - **{e: "js" for e in (".js", ".jsx", ".mjs", ".ejs", ".ts", ".tsx", ".mts", ".cts", ".vue", ".svelte")}, + **{e: "js" for e in (".js", ".jsx", ".mjs", ".cjs", ".ejs", ".ts", ".tsx", ".mts", ".cts", ".vue", ".svelte")}, **{e: "go" for e in (".go",)}, **{e: "rust" for e in (".rs",)}, **{e: "jvm" for e in (".java", ".kt", ".kts", ".scala")}, **{e: "c" for e in (".c", ".h", ".cpp", ".cc", ".cxx", ".hpp")}, - **{e: "ruby" for e in (".rb",)}, + **{e: "ruby" for e in (".rb", ".rake")}, **{e: "swift" for e in (".swift",)}, **{e: "dotnet" for e in (".cs",)}, **{e: "php" for e in (".php",)}, @@ -64,11 +72,12 @@ def _is_file_node(G: nx.Graph, node_id: str) -> bool: label = attrs.get("label", "") if not label: return False - # File-level hub: label matches the actual source filename (not just any label ending in .py) + # File-level hub: label matches the actual source filename — bare basename OR + # the directory-qualified form the #2032 disambiguation pass may assign. source_file = attrs.get("source_file", "") if source_file: - from pathlib import Path as _Path - if label == _Path(source_file).name: + from graphify.build import _is_file_node_label + if _is_file_node_label(label, source_file): return True # Method stub: AST extractor labels methods as '.method_name()' if label.startswith(".") and label.endswith("()"): @@ -504,7 +513,10 @@ def suggest_questions( # 4. Isolated or weakly-connected nodes → exploration questions isolated = [ n for n in G.nodes() - if G.degree(n) <= 1 and not _is_file_node(G, n) and not _is_concept_node(G, n) + if G.degree(n) <= 1 + and not _is_file_node(G, n) + and not _is_concept_node(G, n) + and G.nodes[n].get("file_type") != "rationale" ] if isolated: labels = [G.nodes[n].get("label", n) for n in isolated[:3]] diff --git a/graphify/benchmark.py b/graphify/benchmark.py index ee9c3925e..cf2bc6ba2 100644 --- a/graphify/benchmark.py +++ b/graphify/benchmark.py @@ -1,10 +1,7 @@ """Token-reduction benchmark - measures how much context graphify saves vs naive full-corpus approach.""" from __future__ import annotations -import json import sys -from pathlib import Path import networkx as nx -from networkx.readwrite import json_graph from graphify.build import edge_data from graphify.serve import _query_terms @@ -100,13 +97,11 @@ def run_benchmark( Returns dict with: corpus_tokens, avg_query_tokens, reduction_ratio, per_question """ graph_path = graph_path or _default_graph_json() - from graphify.security import check_graph_file_size_cap - check_graph_file_size_cap(Path(graph_path)) - data = json.loads(Path(graph_path).read_text(encoding="utf-8")) - try: - G = json_graph.node_link_graph(data, edges="links") - except TypeError: - G = json_graph.node_link_graph(data) + # Size-cap check + links/edges normalization + node-link parse. A raw + # --no-cluster graph stores edges under "edges" and used to KeyError + # here (#2212). + from graphify.paths import load_node_link_graph + G = load_node_link_graph(graph_path) if corpus_words is None: # Rough estimate: each node label is ~3 words, plus source context diff --git a/graphify/build.py b/graphify/build.py index 279eccbf4..1f840bbc0 100644 --- a/graphify/build.py +++ b/graphify/build.py @@ -22,6 +22,7 @@ # from __future__ import annotations import json +import math import os import re import sys @@ -33,6 +34,26 @@ from .validate import validate_extraction +# Language interop families, keyed by extension, for the cross-language phantom-edge +# guard in the edge loop below. Families group by REAL interop (JS/TS share a module +# graph; C/C++/ObjC share a compilation unit via headers; JVM langs share bytecode), +# so a legitimate TS->JS import or C impl->header call survives, while a Python +# `import time` binding to a `time.ts` (#1749) or a cross-language INFERRED `calls` +# edge (#1547/#1556) is dropped. Kept local to build.py (not imported from extract.py, +# which imports build.py — a cycle) and deliberately mirrors extract._LANG_FAMILY_BY_EXT. +_EDGE_LANG_FAMILY: dict[str, str] = { + ".py": "py", ".pyi": "py", + ".js": "js", ".mjs": "js", ".cjs": "js", ".jsx": "js", + ".ts": "js", ".tsx": "js", ".mts": "js", ".cts": "js", + ".go": "go", ".rs": "rs", + ".java": "jvm", ".kt": "jvm", ".scala": "jvm", ".groovy": "jvm", + ".c": "c", ".h": "c", ".cc": "c", ".cpp": "c", ".hpp": "c", + ".cxx": "c", ".hh": "c", ".hxx": "c", + ".cu": "c", ".cuh": "c", ".metal": "c", ".m": "c", ".mm": "c", + ".rb": "rb", ".rake": "rb", ".php": "php", ".cs": "cs", ".swift": "swift", ".lua": "lua", +} + + # Synonym mapper for known invalid file_type values that LLM subagents commonly # emit. Keeps semantic intent close (markdown→document, tool→code) and falls # back to "concept" for any other invalid value (see #840). @@ -103,6 +124,39 @@ def _normalize_hyperedge_members(he: object) -> None: he.pop(alias, None) +def _fold_node_aliases(node: dict) -> None: + """Fold legacy node field aliases onto canonical keys, in place (#2194). + + ``name`` -> ``label`` and ``path`` -> ``source_file``. Uses an empty-check + (not mere key presence) so a node carrying ``label: ""``/``None`` next to a + real ``name`` is healed too. When the canonical field already holds a value + it wins and the alias key is left untouched. Without this fold an alias-only + node enters the graph with no label/source_file: it fails validation, gets + ``norm_label == ""`` (invisible to query/explain), and is excluded from every + label-keyed merge/dedup — a permanent ghost that ``graphify update`` + re-feeds through build_from_json forever. + """ + if not node.get("label") and isinstance(node.get("name"), str) and node["name"]: + node["label"] = node.pop("name") + if not node.get("source_file") and isinstance(node.get("path"), str) and node["path"]: + node["source_file"] = node.pop("path") + + +def _fold_edge_aliases(edge: dict) -> None: + """Fold legacy edge field aliases onto canonical keys, in place (#2194). + + ``type`` -> ``relation``. A ``confidence_score`` float with no ``confidence`` + enum backfills ``confidence: "INFERRED"`` — never EXTRACTED (alias recovery + is not provenance) and never a threshold mapping of the float. The + ``confidence_score`` key itself is NOT popped: it is a legitimate companion + field that the edge loop sanitizes and to_json round-trips. + """ + if not edge.get("relation") and isinstance(edge.get("type"), str) and edge["type"]: + edge["relation"] = edge.pop("type") + if not edge.get("confidence") and edge.get("confidence_score") is not None: + edge["confidence"] = "INFERRED" + + def _norm_source_file(p: str | None, root: str | None = None) -> str | None: """Normalize path separators and relativize absolute paths. @@ -130,6 +184,102 @@ def _norm_source_file(p: str | None, root: str | None = None) -> str | None: return p +def _abs_identity(p: str | None, root: str | None = None) -> str | None: + """Return a form-insensitive absolute identity for a source_file. + + prune/replace matching in build_merge otherwise compares raw strings against + ``_norm_source_file`` output, so a node whose source_file survived in a THIRD + form — absolute where prune_sources is relative, or vice versa, or a symlinked + root — slips past every equality check and its nodes/edges are never pruned + (silent survival of a deleted file's graph, #2012). Anchoring relative paths + at ``root`` and resolving both sides to a canonical absolute posix path gives + a fallback that matches regardless of which form each side happens to hold. + """ + if not p: + return None + q = p.replace("\\", "/") + pp = Path(q) + if not pp.is_absolute() and root: + pp = Path(root) / q + try: + return pp.resolve().as_posix() + except OSError: + return pp.as_posix() + + +def _is_file_node_label(label: "str | None", source_file: "str | None") -> bool: + """Whether *label* is a file node's label for *source_file* — the bare + basename, OR a directory-qualified suffix produced by the disambiguation pass + below (#2032). Used both to recognize file nodes when relabeling and by the + downstream file-node predicates (analyze/tree/serve).""" + if not label or not source_file: + return False + sf = str(source_file).replace("\\", "/") + lbl = str(label) + if lbl == sf.rsplit("/", 1)[-1]: + return True + return "/" in lbl and (sf == lbl or sf.endswith("/" + lbl)) + + +def _shortest_unique_suffix(sf: str, all_sfs: "set[str]") -> str: + """Shortest trailing path suffix (basename + k parent dirs) of *sf* that is + unique among *all_sfs*. `a/b/index.ts` vs `c/b/index.ts` -> `a/b/index.ts`; + `x/index.ts` vs `y/index.ts` -> `x/index.ts`. Derived from the path (never the + current label) so relabeling is idempotent across incremental rebuilds.""" + parts = [p for p in sf.replace("\\", "/").split("/") if p] + others = [ + [p for p in o.replace("\\", "/").split("/") if p] + for o in all_sfs if o != sf + ] + for k in range(1, len(parts) + 1): + suffix = parts[-k:] + if all(o[-k:] != suffix for o in others): + return "/".join(suffix) + return "/".join(parts) + + +def _file_label_reassignments(items: "list[tuple]") -> dict: + """Given (key, label, source_file) triples, return {key: new_label} for file + nodes whose basename collides with another's — the shortest unique + directory-qualified suffix (#2032). Keys of non-colliding/basename-unique file + nodes are omitted (their label stays bare).""" + from collections import defaultdict + groups: dict[str, list[tuple]] = defaultdict(list) + for key, label, sf in items: + if sf and label and _is_file_node_label(str(label), str(sf)): + basename = str(sf).replace("\\", "/").rsplit("/", 1)[-1] + groups[basename].append((key, str(sf))) + out: dict = {} + for members in groups.values(): + distinct = {sf for _, sf in members} + if len(distinct) < 2: + continue # no collision — leave the bare basename label + for key, sf in members: + out[key] = _shortest_unique_suffix(sf, distinct) + return out + + +def _disambiguate_file_node_labels(G: "nx.Graph") -> None: + """Relabel colliding-basename file nodes on a graph (#2032). Ids/edges are + never changed — only display labels. Idempotent (labels derive from + source_file, not the current possibly-qualified label).""" + items = [(nid, a.get("label"), a.get("source_file")) for nid, a in G.nodes(data=True)] + for nid, new_label in _file_label_reassignments(items).items(): + G.nodes[nid]["label"] = new_label + + +def disambiguate_file_labels_in_nodes(nodes: "list") -> None: + """Relabel colliding-basename file nodes on a raw node-dict list, in place + (#2032). Used by the extract --no-cluster path, which writes the merged + extraction directly without going through build_from_json.""" + items = [ + (i, n.get("label"), n.get("source_file")) + for i, n in enumerate(nodes) if isinstance(n, dict) + ] + for i, new_label in _file_label_reassignments(items).items(): + nodes[i]["label"] = new_label + + def _infer_merge_root(graph_path: Path) -> str | None: """Best-effort scan root for relativizing paths in build_merge when the caller passes no ``root`` (#1571). @@ -267,8 +417,34 @@ def _semantic_id_remap(nodes: list, root: str | None) -> dict: if not new_stem: continue norm_nid = _normalize_id(nid) + # Idempotency guard (#1917): an id already carrying its canonical stem is + # done — do not re-run the legacy branch on it. When the canonical stem + # contains a shorter legacy stem as a prefix (parent dir name == file + # stem, e.g. `.claude/CLAUDE.md` -> `claude_claude` over legacy `claude`), + # an already-migrated id like `claude_claude_x` still matches the legacy + # `claude_` prefix below and would gain another stem segment on every + # build, defeating the same_topology/no_change short-circuits. Mirrors the + # canonical check in graph_has_legacy_ids. + if norm_nid == new_stem or norm_nid.startswith(new_stem + "_"): + continue new_id: str | None = None - for old_stem in _old_file_stems(rel): + old_forms = _old_file_stems(rel) + # #2197: on Windows, detect() can emit an ABSOLUTE source_file, and a + # semantic fragment's id derived from that absolute path (e.g. + # d_projects_myrepo_docs_dataflow) matches neither the canonical + # relative stem nor the legacy short forms above — so while source_file + # itself is healed by _norm_source_file, the id would ghost against the + # existing graph's docs_dataflow. When the raw path was absolute and + # relativized under root, treat the raw-absolute stem as one more + # old-stem form — the semantic-side twin of extract.py's absolute-form + # id registration. It is the longest form, so it goes first (greedy + # prefix stripping, same ordering rule as _old_file_stems). + sf_raw = str(sf).replace("\\", "/") + if sf_raw != sf_norm and os.path.isabs(sf_raw): + abs_stem = make_id(_file_stem(Path(sf_raw))) + if abs_stem and abs_stem != new_stem and abs_stem not in old_forms: + old_forms.insert(0, abs_stem) + for old_stem in old_forms: if old_stem == new_stem: continue # already canonical for this form if norm_nid == old_stem: @@ -328,6 +504,38 @@ def graph_has_legacy_ids(nodes: list, root: str | Path | None = None, sample: in return False +def _doc_twin_remap(nodes: list) -> dict[str, str]: + """Map a markdown quick-scan's bare doc node ```` to the semantic + ``_doc`` node for the SAME file (#1799). + + The markdown quick-scan (``extract_markdown``) mints a file node with the + bare id ``_make_id(path)`` while the semantic pass mints ``_doc`` for + the same document. A ``graphify update`` after a semantic build leaves both, + splitting the file's edges across two disconnected nodes. Canonicalize to the + semantic ``_doc`` node (it carries the richer references/hyperedges). Gated to + ``file_type == "document"`` on BOTH twins with an identical ``source_file``, + so an unrelated code symbol ``foo`` and ``foo_doc`` never merge. + """ + by_id: dict[str, dict] = {} + for n in nodes: + if isinstance(n, dict) and n.get("id"): + by_id[str(n["id"])] = n + remap: dict[str, str] = {} + for nid, node in by_id.items(): + if not nid.endswith("_doc"): + continue + bare = by_id.get(nid[:-4]) + if bare is None: + continue + sf = node.get("source_file") + if not sf or bare.get("source_file") != sf: + continue + if node.get("file_type") != "document" or bare.get("file_type") != "document": + continue + remap[nid[:-4]] = nid + return remap + + def build_from_json(extraction: dict, *, directed: bool = False, root: str | Path | None = None) -> nx.Graph: """Build a NetworkX graph from an extraction dict. @@ -359,6 +567,11 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat file=sys.stderr, ) node["source_file"] = node.pop("source") + # Fold the remaining legacy node aliases (`name`->`label`, + # `path`->`source_file`, #2194) before validation and before the + # semantic-rekey / ghost-merge passes below, all of which key on + # label/source_file and would otherwise skip the node entirely. + _fold_node_aliases(node) # Default missing/None file_type to "concept" so legacy graph.json # entries (and stub nodes preserved by `_rebuild_code` from older # graphify versions that didn't always populate file_type) don't @@ -377,11 +590,33 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat for he in extraction.get("hyperedges", []) or []: _normalize_hyperedge_members(he) + # Fold legacy edge field aliases (`type`->`relation`, + # `confidence_score`->`confidence`, #2194) BEFORE validation. The existing + # from/to endpoint fold lives in the edge loop further down, which runs + # after validate_extraction — too late for fields the validator requires. + for edge in extraction.get("edges", []): + if isinstance(edge, dict): + _fold_edge_aliases(edge) + errors = validate_extraction(extraction) # Dangling edges (stdlib/external imports) are expected - only warn about real schema errors. real_errors = [e for e in errors if "does not match any node id" not in e] if real_errors: - print(f"[graphify] Extraction warning ({len(real_errors)} issues): {real_errors[0]}", file=sys.stderr) + # Break the warning down by cause (#2194): a mixed batch used to surface + # only real_errors[0], hiding every other failure mode. Group on the + # "missing required field 'X'" suffix and report per-cause counts plus + # one example each, so the operator sees the full shape of the damage. + by_cause: dict[str, list[str]] = {} + for err in real_errors: + m = re.search(r"missing required field '[^']*'", err) + by_cause.setdefault(m.group(0) if m else "other schema issue", []).append(err) + breakdown = "; ".join( + f"{len(errs)}x {cause} (e.g. {errs[0]})" for cause, errs in by_cause.items() + ) + print( + f"[graphify] Extraction warning ({len(real_errors)} issues): {breakdown}", + file=sys.stderr, + ) # Deterministic semantic re-key (#1504/#1509): the node-ID stem is now the # full repo-relative path (docs/v1/api/README.md -> docs_v1_api_readme), but # the semantic cache is UNVERSIONED, so a cached/LLM fragment can still carry @@ -408,6 +643,33 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat if isinstance(he, dict) and isinstance(he.get("nodes"), list): he["nodes"] = [_rekey.get(n, n) for n in he["nodes"]] + # Merge markdown quick-scan bare doc nodes into their semantic `_doc` twin + # for the same file, so a document is one node regardless of which pipeline + # touched it last (#1799). + _doc_remap = _doc_twin_remap(extraction.get("nodes", [])) + if _doc_remap: + extraction["nodes"] = [ + n for n in extraction.get("nodes", []) + if not (isinstance(n, dict) and n.get("id") in _doc_remap) + ] + _new_edges = [] + for edge in extraction.get("edges", []): + if isinstance(edge, dict): + s0, t0 = edge.get("source"), edge.get("target") + if s0 in _doc_remap: + edge["source"] = _doc_remap[s0] + if t0 in _doc_remap: + edge["target"] = _doc_remap[t0] + # Drop only self-loops the remap itself collapsed (a bare->_doc + # link becoming doc->doc); leave any pre-existing self-loop alone. + if edge.get("source") == edge.get("target") and (s0 in _doc_remap or t0 in _doc_remap): + continue + _new_edges.append(edge) + extraction["edges"] = _new_edges + for he in extraction.get("hyperedges", []) or []: + if isinstance(he, dict) and isinstance(he.get("nodes"), list): + he["nodes"] = [_doc_remap.get(n, n) for n in he["nodes"]] + G: nx.Graph = nx.DiGraph() if directed else nx.Graph() for node in extraction.get("nodes", []): # Skip dict nodes with a missing or non-hashable id (e.g. a list emitted @@ -439,9 +701,9 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # populates source_location, so those ghosts survived. Extended fix: use # _origin=="ast" as the canonical signal. AST nodes always win; any non-AST # node sharing (basename, label) with an AST node is a ghost. - _loc_nodes: dict[tuple[str, str], str] = {} # (basename, label) -> canonical node id + _loc_nodes: dict[tuple[str, str], str] = {} # (source_file, label) -> canonical node id _loc_collisions: set[tuple[str, str]] = set() # keys shared by 2+ AST nodes - _noloc_nodes: dict[tuple[str, str], str] = {} # (basename, label) -> ghost node id + _noloc_nodes: dict[tuple[str, str], str] = {} # (source_file, label) -> ghost node id # Pass 1: collect canonical nodes — AST-origin nodes take precedence over LLM nodes. # When 2+ AST nodes share a key (same-named symbols in same-named files across @@ -449,36 +711,50 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # ghost would pick an arbitrary winner via set-iteration order (#1257). Track # those keys so Pass 2 skips them — same conservatism as # _rewire_unique_stub_nodes, which only merges when exactly one real def exists. - for nid in node_set: + # Iterate in a deterministic (sorted) order, not set-iteration order, so the + # canonical winner and the ambiguity decisions below don't flip run-to-run + # with CPython's per-process string-hash seed (#1753) — the same reason the + # edge-iteration loop further down sorts on purpose. + for nid in sorted(node_set): attrs = G.nodes[nid] label = str(attrs.get("label", "")).strip() sf = str(attrs.get("source_file", "")) - basename = Path(sf).name if sf else "" - if not label or not basename: + if not label or not sf: continue is_ast = attrs.get("_origin") == "ast" if attrs.get("source_location") or is_ast: - key = (basename, label) + # Key on the FULL normalized source_file, not the bare basename + # (#2068): the AST/LLM ghost twins of #1145 always share the same + # source_file (different ids, same file), so full-path keying still + # collapses them, while unrelated same-basename nodes in DIFFERENT + # directories (docs/a/index.md vs docs/b/index.md) now get distinct + # keys and are never falsely merged. This subsumes the #1753/#1257 + # cross-file ambiguity guard, which is why the non-AST branch below + # no longer needs it. + key = (sf, label) if is_ast: - # Two AST nodes on the same key is an ambiguous collision. + # Two AST nodes on the same key (same file, same label) is an + # ambiguous collision. if key in _loc_nodes and G.nodes[_loc_nodes[key]].get("_origin") == "ast": _loc_collisions.add(key) # AST-origin nodes always overwrite a prior non-AST entry. _loc_nodes[key] = nid - elif key not in _loc_nodes: - _loc_nodes[key] = nid + else: + # First non-AST node for this (file, label) wins as canonical; a + # later same-key node is a genuine same-file duplicate and still + # collapses in Pass 2. + _loc_nodes.setdefault(key, nid) # Pass 2: find ghosts — non-AST nodes that have an AST canonical twin. - for nid in node_set: + for nid in sorted(node_set): attrs = G.nodes[nid] if attrs.get("_origin") == "ast": continue # AST nodes are never ghosts label = str(attrs.get("label", "")).strip() sf = str(attrs.get("source_file", "")) - basename = Path(sf).name if sf else "" - if not label or not basename: + if not label or not sf: continue - key = (basename, label) + key = (sf, label) if key in _loc_collisions: continue # ambiguous key: no safe canonical winner, leave ghost intact if key in _loc_nodes and _loc_nodes[key] != nid: @@ -507,7 +783,32 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # fragment (e.g. an incremental update whose fragment references a symbol in a # file that was NOT re-extracted) still resolves to the migrated node instead # of dangling. Only fills gaps — never overrides a real node id. + # + # The old-stem form drops the extension and (for the file node itself) every + # directory but the immediate parent, so it collapses easily: "ping.h" and + # "ping.php" in different directories both alias to bare "ping". Collecting + # every candidate for an alias BEFORE committing any of them — and only + # committing when exactly one candidate claims it — keeps this a precise + # re-keying aid instead of a silent cross-file (and cross-language) merge. + # Without this, a dangling edge to a bare, deliberately-unscoped fallback id + # (e.g. the C/C++ extractor's last-resort target for an #include it couldn't + # resolve to a real path) could ride this alias onto whichever unrelated + # same-stem file happened to be inserted first into ``node_set`` — a Python + # set, so "first" is hash-order, not anything meaningful. + # + # A file node's OWN id is not always a clean ``new_stem`` prefix: when a + # same-directory ``.h``/``.cpp`` pair collides on their shared pre-extension + # id, _disambiguate_colliding_node_ids salts both apart into ids like + # ``tools_aolserver_utility_h_tools_aolserver_utility`` — which no longer + # string-prefixes cleanly for the suffix math below. Detecting "this IS the + # file node" by label (every file node's label is its own basename, + # regardless of id mangling) instead of by id shape keeps a salted file node + # in the alias competition, so a genuine collision (a C header AND an + # unrelated same-named PHP script) is still caught as ambiguous instead of + # the header silently dropping out of the race and leaving the PHP file as + # the lone (wrong) "unambiguous" winner. from graphify.extractors.base import _file_stem as _fs + _alias_candidates: dict[str, set[str]] = {} for nid in node_set: attrs = G.nodes[nid] sf = attrs.get("source_file") @@ -517,15 +818,21 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat if rel.is_absolute(): continue new_stem = make_id(_fs(rel)) - suffix = "" - if _normalize_id(nid).startswith(new_stem): - suffix = _normalize_id(nid)[len(new_stem):] # leading "_entity" or "" + if str(attrs.get("label", "")) == rel.name: + suffix = "" # this node IS the file, whatever its (possibly salted) id + else: + suffix = "" + if _normalize_id(nid).startswith(new_stem): + suffix = _normalize_id(nid)[len(new_stem):] # leading "_entity" or "" for old_stem in _old_file_stems(rel): if old_stem == new_stem: continue alias = old_stem + suffix - norm_to_id.setdefault(_normalize_id(alias), nid) - norm_to_id.setdefault(alias, nid) + _alias_candidates.setdefault(_normalize_id(alias), set()).add(nid) + _alias_candidates.setdefault(alias, set()).add(nid) + for alias_key, candidates in _alias_candidates.items(): + if len(candidates) == 1: + norm_to_id.setdefault(alias_key, next(iter(candidates))) # Iterate edges in a deterministic order. The graph is undirected and stores # direction in _src/_tgt; when two edges collapse onto the same node pair the # last write wins, so an unstable iteration order flips _src/_tgt run-to-run @@ -565,7 +872,36 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat tgt = norm_to_id.get(_normalize_id(tgt), tgt) if src not in node_set or tgt not in node_set: continue # skip edges to external/stdlib nodes - expected, not an error - attrs = {k: v for k, v in edge.items() if k not in ("source", "target")} + # `target_file` is a transient import-disambiguation salt hint (#1814) + # with no downstream reader; it holds an absolute path, so it must never + # be persisted. Disambiguation already pops it off fresh extractions — + # dropping it here as well keeps a pre-fix graph's stale absolute hint + # from surviving an incremental build_merge, which re-serializes base + # edges through here without re-running disambiguation. + # `local_alias` is the same shape of transient hint (#2082): it exists only + # for the module arm of _resolve_python_member_calls to match an aliased + # import receiver, and extract() already drops it once that pass has run. + # Dropping it here too covers a stale pre-fix graph re-serialized through + # an incremental build_merge, same rationale as target_file above. + # Sanitize numeric edge fields (#1960): an explicit ``"weight": null`` in + # the extraction JSON survives ``.get("weight", 1.0)`` (the key is present, + # so the default never applies) and reaches Louvain/Leiden as None, + # crashing modularity arithmetic with a TypeError (graspologic's Leiden + # even panics on NaN). Coerce to float and fall back to the schema default + # of 1.0 for anything the clustering backends reject — None, non-numeric + # strings, NaN/inf, negatives — while numeric strings coerce cleanly. + # Repair (not drop) the key so graph.json round-trips a clean value and a + # cluster-only/--update reload never re-ingests the null. + attrs = {k: v for k, v in edge.items() if k not in ("source", "target", "target_file", "local_alias")} + for _num_key in ("weight", "confidence_score"): + if _num_key in attrs: + try: + _num_val = float(attrs[_num_key]) + except (TypeError, ValueError): + _num_val = 1.0 + if not math.isfinite(_num_val) or _num_val < 0: + _num_val = 1.0 + attrs[_num_key] = _num_val # Backfill source_file from the endpoint nodes (every node carries one). # Semantic/LLM edges occasionally omit it, which downstream validation # flags and leaves query results with no file reference (#1279). @@ -577,31 +913,41 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat ) if "source_file" in attrs: attrs["source_file"] = _norm_source_file(attrs["source_file"], _root) - # Drop cross-language INFERRED `calls` edges — same short names (render, - # parse, etc.) appear across language boundaries in multi-language chunks, - # producing phantom edges that don't represent real call relationships. - if attrs.get("relation") == "calls" and attrs.get("confidence") == "INFERRED": - _LANG_FAMILY: dict[str, str] = { - ".py": "py", ".pyi": "py", - ".js": "js", ".mjs": "js", ".cjs": "js", ".jsx": "js", - ".ts": "js", ".tsx": "js", ".mts": "js", ".cts": "js", - ".go": "go", ".rs": "rs", - ".java": "jvm", ".kt": "jvm", ".scala": "jvm", ".groovy": "jvm", - # C, C++, and ObjC interoperate within one compilation unit: a method - # declared in a shared `.h` is defined/called from a `.c`/`.cpp`/`.m` - # sibling, so a cross-file INFERRED call from impl to its header decl - # is legitimate, not a phantom name-collision across languages. Treat - # the whole C family as one so the receiver-typed C++/ObjC member-call - # resolvers' header-targeting edges survive build (#1547/#1556). - ".c": "c", ".h": "c", ".cc": "c", ".cpp": "c", ".hpp": "c", - ".cxx": "c", ".hh": "c", ".hxx": "c", - ".cu": "c", ".cuh": "c", ".metal": "c", ".m": "c", ".mm": "c", - ".rb": "rb", ".php": "php", ".cs": "cs", ".swift": "swift", ".lua": "lua", - } + # Drop cross-language phantom edges — the same short names (render, parse, + # time, ...) recur across language boundaries, so an unresolved target can + # bind to a same-named node in another language. The extraction spec forbids + # this for `calls`; it is equally invalid for `imports`/`references` (a + # Python `import time` must not bind to a `time.ts`, #1749). + _edge_rel = attrs.get("relation") + if _edge_rel in ("calls", "imports", "imports_from", "references"): src_ext = Path(G.nodes[src].get("source_file") or "").suffix.lower() tgt_ext = Path(G.nodes[tgt].get("source_file") or "").suffix.lower() - if src_ext and tgt_ext and _LANG_FAMILY.get(src_ext) != _LANG_FAMILY.get(tgt_ext): - continue + src_fam = _EDGE_LANG_FAMILY.get(src_ext) + tgt_fam = _EDGE_LANG_FAMILY.get(tgt_ext) + if _edge_rel == "calls": + # Unchanged #1547/#1556 behavior: only INFERRED calls, and drop as + # soon as either family differs (an unknown ext counts as different). + if ( + attrs.get("confidence") == "INFERRED" + and src_ext and tgt_ext and src_fam != tgt_fam + ): + continue + else: + # imports/references: drop only when BOTH endpoints are known code + # languages of different families, so a config->code reference + # (unknown ext, e.g. a manifest) is never mistaken for a phantom. + if src_fam is not None and tgt_fam is not None and src_fam != tgt_fam: + continue + # A file-level import or re-export cannot carry useful connectivity when + # both endpoints resolve to the same node. This most often happens when + # the target is an unresolved bare module name (``builtins``, ``poseidon``) + # that the legacy-ID alias index above mistakes for the importing file's + # own old stem. It also covers a nested module importing its parent file: + # at file-node granularity that relationship necessarily collapses. Keep + # other self-edges, notably recursive ``calls``, because those are real + # program structure rather than import-resolution artifacts. + if src == tgt and _edge_rel in ("imports", "imports_from", "re_exports"): + continue # Preserve original edge direction - undirected graphs lose it otherwise, # causing display functions to show edges backwards. attrs["_src"] = src @@ -625,10 +971,48 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat # Relativize hyperedge source_file the same way nodes and edges are # (above), so to_json — which has no root and writes G.graph["hyperedges"] # verbatim — never leaks an absolute path from a semantic subagent (#1418). + kept_hyperedges = [] for he in hyperedges: if isinstance(he, dict) and he.get("source_file"): he["source_file"] = _norm_source_file(he["source_file"], _root) - G.graph["hyperedges"] = hyperedges + # Validate members against the built node set (#1916): a hyperedge + # member absent from the graph used to be copied into + # G.graph["hyperedges"] verbatim and reach graph.json dangling, + # even from a live (non-cache) extraction. Mirror the pairwise-edge + # handling above: remap mismatched ids via normalization first, + # then drop members that still don't resolve; drop the hyperedge + # itself when no valid member remains (single-member hyperedges + # are legal in this codebase, e.g. a per-file flow, so we prune + # rather than require two survivors). + if isinstance(he, dict) and isinstance(he.get("nodes"), list): + valid_members = [] + for m in he["nodes"]: + try: + hash(m) + except TypeError: + continue + if m not in node_set and isinstance(m, str): + m = norm_to_id.get(_normalize_id(m), m) + if m in node_set: + valid_members.append(m) + if not valid_members: + print( + f"[graphify] WARNING: dropping hyperedge " + f"{he.get('id', '?')!r} — none of its members " + f"{he.get('nodes')!r} match built nodes.", + file=sys.stderr, + ) + continue + if valid_members != he["nodes"]: + he["nodes"] = valid_members + kept_hyperedges.append(he) + if kept_hyperedges: + G.graph["hyperedges"] = kept_hyperedges + # Runs LAST, after the alias-competition above (which relies on file-node + # labels still being bare basenames): give colliding-basename file nodes a + # directory-qualified display label so lookup/discovery can disambiguate + # them (#2032). Labels only — ids and edges are untouched. + _disambiguate_file_node_labels(G) return G @@ -649,10 +1033,11 @@ def build( ambiguous pairs in the 75–92 Jaro-Winkler score zone. root: if given, absolute source_file paths are made relative to root (#932). - Extractions are merged in order. For nodes with the same ID, the last - extraction's attributes win (NetworkX add_node overwrites). Pass AST - results before semantic results so semantic labels take precedence, or - reverse the order if you prefer AST source_location precision to win. + With dedup disabled, extractions are merged in order and the last node's + attributes win (NetworkX add_node overwrites). With dedup enabled, nodes + sharing an ID use a deterministic survivor and retain missing attributes + from duplicate records of the same source entity. Genuine cross-file ID + collisions remain isolated and are reported. """ from graphify.dedup import deduplicate_entities combined: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} @@ -663,6 +1048,13 @@ def build( combined["input_tokens"] += ext.get("input_tokens", 0) combined["output_tokens"] += ext.get("output_tokens", 0) if dedup and combined["nodes"]: + # Fold legacy node field aliases before dedup (#2194): dedup runs BEFORE + # build_from_json and keys on `label`, so a `name`/`path` alias node + # would be invisible to it and only label-dedup one build later, after + # build_from_json's own fold has healed the persisted graph.json. + for n in combined["nodes"]: + if isinstance(n, dict): + _fold_node_aliases(n) combined["nodes"], combined["edges"] = deduplicate_entities( combined["nodes"], combined["edges"], communities={}, dedup_llm_backend=dedup_llm_backend, @@ -732,6 +1124,137 @@ def deduplicate_by_label(nodes: list[dict], edges: list[dict]) -> tuple[list[dic return deduped_nodes, deduped_edges +def _load_existing_graph(graph_path: Path) -> "tuple[list, list, list] | None": + """Load (nodes, edges, hyperedges) from an existing graph.json for an + incremental merge, accepting both the ``links`` and ``edges`` spellings. + + Reads the JSON directly instead of going through node_link_graph(). + The latter rebuilds an undirected nx.Graph and then enumerating + edges() yields endpoints based on node insertion order, which + silently flips directional edges (e.g. `calls`) when the callee + was inserted before the caller. The _src/_tgt direction-preserving + attrs are popped before saving in export.py, so going through the + NetworkX round-trip loses direction permanently (#760). + + Returns None when the file does not exist. Raises RuntimeError when it + exists but cannot be parsed — callers must refuse to overwrite rather + than silently replace a possibly-recoverable graph. + """ + if not graph_path.exists(): + return None + from graphify.security import check_graph_file_size_cap + check_graph_file_size_cap(graph_path) + try: + data = json.loads(graph_path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as exc: + raise RuntimeError( + f"Cannot read {graph_path} for incremental merge: {exc}. " + "Delete the file and run a full rebuild." + ) from exc + links_key = "links" if "links" in data else "edges" + return ( + list(data.get("nodes", [])), + list(data.get(links_key, [])), + list(data.get("hyperedges", [])), + ) + + +def merge_raw_extraction( + new: dict, + graph_path: str | Path, + prune_sources: "list[str] | None" = None, + root: "str | Path | None" = None, +) -> dict: + """Merge the existing raw graph.json forward into a fresh raw extraction + (the ``extract --no-cluster`` incremental path, #2169). + + Replace/prune semantics mirror :func:`build_merge` exactly, so the raw and + clustered incremental paths can't drift: + + - sources re-extracted this run REPLACE their prior contribution — existing + nodes/edges/hyperedges owned by them are dropped, matched in both raw and + :func:`_norm_source_file` form (#1007); + - ``prune_sources`` (deleted / excluded / graph-stale files) are dropped, + with the ``_abs_identity`` third-form fallback (#2012), and "replace" wins + over a contradictory "delete" of a re-extracted source (#1796); + - everything else — nodes/edges/hyperedges owned by unchanged files — is + carried forward unchanged. + + Survivors are PREPENDED to ``new``'s lists (existing-first), so the caller's + ``dedupe_nodes`` last-writer-wins keeps fresh attributes for re-extracted + nodes while ``dedupe_edges`` first-wins never resurrects a replaced edge + (replaced sources' edges were already dropped above). Token counters and + every other key of ``new`` are left untouched. Returns ``new``, mutated in + place. Raises RuntimeError (via :func:`_load_existing_graph`) when the + existing graph is present but unparseable — the caller must refuse to + overwrite it. No-op when ``graph_path`` does not exist. + """ + graph_path = Path(graph_path) + loaded = _load_existing_graph(graph_path) + if loaded is None: + return new + existing_nodes, existing_edges, existing_hyperedges = loaded + + _eff_root = ( + str(Path(root).resolve()) if root is not None + else _infer_merge_root(graph_path) + ) + + new_sources: set[str] = set() + for n in new.get("nodes", []): + if not isinstance(n, dict): + continue + sf = n.get("source_file") + if not sf: + continue + new_sources.add(sf) + norm = _norm_source_file(sf, _eff_root) + if norm: + new_sources.add(norm) + + prune_set: set[str] = set() + prune_abs: set[str] = set() + for p in (prune_sources or []): + if not p: + continue + prune_set.add(p) + norm = _norm_source_file(p, _eff_root) + if norm: + prune_set.add(norm) + a = _abs_identity(p, _eff_root) + if a: + prune_abs.add(a) + # "Replace" wins over a contradictory "delete" of the same source (#1796), + # in both string and absolute-identity space (#2012) — as in build_merge. + prune_set -= new_sources + new_abs = {_abs_identity(s, _eff_root) for s in new_sources} + new_abs.discard(None) + prune_abs -= new_abs + + def _dropped(item: dict) -> bool: + if not isinstance(item, dict): + return True + sf = item.get("source_file") + if sf in new_sources or _norm_source_file(sf, _eff_root) in new_sources: + return True # re-extracted this run — replaced by the new chunk + if not sf: + return False # unowned — carry forward + if sf in prune_set: + return True + norm = _norm_source_file(sf, _eff_root) + if norm and norm in prune_set: + return True + a = _abs_identity(sf, _eff_root) + return bool(a) and a in prune_abs + + new["nodes"] = [n for n in existing_nodes if not _dropped(n)] + list(new.get("nodes", [])) + new["edges"] = [e for e in existing_edges if not _dropped(e)] + list(new.get("edges", [])) + carried_hyper = [he for he in existing_hyperedges if not _dropped(he)] + if carried_hyper or new.get("hyperedges"): + new["hyperedges"] = carried_hyper + list(new.get("hyperedges", [])) + return new + + def build_merge( new_chunks: list[dict], graph_path: str | Path | None = None, @@ -752,27 +1275,9 @@ def build_merge( root: if given, absolute source_file paths in new_chunks are made relative (#932). """ graph_path = Path(graph_path if graph_path is not None else _default_graph_json()) - if graph_path.exists(): - # Read JSON directly instead of going through node_link_graph(). - # The latter rebuilds an undirected nx.Graph and then enumerating - # edges() yields endpoints based on node insertion order, which - # silently flips directional edges (e.g. `calls`) when the callee - # was inserted before the caller. The _src/_tgt direction-preserving - # attrs are popped before saving in export.py, so going through the - # NetworkX round-trip loses direction permanently (#760). - from graphify.security import check_graph_file_size_cap - check_graph_file_size_cap(graph_path) - try: - data = json.loads(graph_path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError) as exc: - raise RuntimeError( - f"Cannot read {graph_path} for incremental merge: {exc}. " - "Delete the file and run a full rebuild." - ) from exc - links_key = "links" if "links" in data else "edges" - existing_nodes = list(data.get("nodes", [])) - existing_edges = list(data.get(links_key, [])) - existing_hyperedges = list(data.get("hyperedges", [])) + _loaded = _load_existing_graph(graph_path) + if _loaded is not None: + existing_nodes, existing_edges, existing_hyperedges = _loaded had_graph = True else: existing_nodes = [] @@ -830,6 +1335,7 @@ def _kept(item: dict) -> bool: # handles symlinked roots and ".." / "./" segments so Path.relative_to() # succeeds even when the scan root is a symlink. (#1007, #1571) prune_set: set[str] = set() + prune_abs: set[str] = set() for p in (prune_sources or []): if not p: continue @@ -837,6 +1343,35 @@ def _kept(item: dict) -> bool: norm = _norm_source_file(p, _eff_root) if norm: prune_set.add(norm) + a = _abs_identity(p, _eff_root) + if a: + prune_abs.add(a) + # A file that was just re-extracted (present in new_chunks) is being REPLACED, + # never deleted — so never prune it, even if the caller also lists it in + # prune_sources. Otherwise its fresh, just-built nodes are silently removed + # (data loss): common when an edit keeps a node's label and the caller follows + # the old edit-workflow of passing the changed file in prune_sources (#1796). + # "replace" wins over a contradictory "delete" of the same source. Applied in + # both string and absolute-identity space so the third-form fallback below + # can't resurrect the delete for a re-extracted file (#2012). + prune_set -= new_sources + new_abs = {_abs_identity(s, _eff_root) for s in new_sources} + new_abs.discard(None) + prune_abs -= new_abs + + def _prune_match(sf: "str | None") -> bool: + # Match a node/edge/hyperedge source_file against the prune set in a + # form-insensitive way: exact string, normalised-relative, then the + # absolute-identity fallback for the third-form case (#2012). + if not sf: + return False + if sf in prune_set: + return True + norm = _norm_source_file(sf, _eff_root) + if norm and norm in prune_set: + return True + a = _abs_identity(sf, _eff_root) + return bool(a) and a in prune_abs # Carry forward hyperedges from files that were neither re-extracted nor # deleted (#1574). build() only sees the new chunks' hyperedges, so without @@ -855,7 +1390,7 @@ def _kept(item: dict) -> bool: norm = _norm_source_file(sf, _eff_root) if sf in new_sources or norm in new_sources: continue # re-extracted — replaced by the new chunk's version - if sf in prune_set or norm in prune_set: + if _prune_match(sf): continue # deleted — pruned carried.append(he) if carried: @@ -866,32 +1401,34 @@ def _kept(item: dict) -> bool: if prune_sources: to_remove = [ n for n, d in G.nodes(data=True) - if d.get("source_file") in prune_set + if _prune_match(d.get("source_file")) ] G.remove_nodes_from(to_remove) n_files = len(prune_sources) n_nodes = len(to_remove) if n_nodes: print( - f"[graphify] Pruned {n_nodes} node(s) from {n_files} deleted source file(s).", + f"[graphify] Pruned {n_nodes} node(s) from {n_files} deleted or " + f"excluded source file(s).", file=sys.stderr, ) edges_to_remove = [ (u, v) for u, v, d in G.edges(data=True) - if d.get("source_file") in prune_set + if _prune_match(d.get("source_file")) ] if edges_to_remove: G.remove_edges_from(edges_to_remove) print( - f"[graphify] Pruned {len(edges_to_remove)} edge(s) from deleted source file(s).", + f"[graphify] Pruned {len(edges_to_remove)} edge(s) from deleted or " + f"excluded source file(s).", file=sys.stderr, ) if not n_nodes and not edges_to_remove: print( - f"[graphify] {n_files} source file(s) deleted since last run — " - f"no matching nodes or edges in graph, already clean.", + f"[graphify] {n_files} source file(s) deleted or excluded since " + f"last run — no matching nodes or edges in graph, already clean.", file=sys.stderr, ) @@ -925,6 +1462,34 @@ def prefix_graph_for_global(G: nx.Graph, repo_tag: str) -> nx.Graph: return H +def distinct_repo_tags(graph_paths: "list[Path]") -> "list[str]": + """Return a unique, human-meaningful repo tag per input graph for merge-graphs. + + The naive tag (the ``graphify-out`` parent dir name) is NOT unique across + inputs: ``src/graphify-out`` and ``frontend/src/graphify-out`` both yield + ``src``. Prefixing both node sets with ``src::`` then makes same-stem nodes + (a backend ``src/app.js`` and a frontend ``App.jsx``, both bare ``app``) + collide, so ``nx.compose`` silently merges two unrelated entities and invents + cross-runtime edges (#1729). Colliding tags are widened with their own parent + dir (``frontend_src``), then an index suffix guarantees uniqueness so no two + graphs ever share a prefix. + """ + repo_dirs = [p.parent.parent for p in graph_paths] # graphify-out/.. → repo dir + tags = [d.name or "repo" for d in repo_dirs] + if len(set(tags)) != len(tags): + widened: list[str] = [] + for d in repo_dirs: + parent = d.parent.name + widened.append(f"{parent}_{d.name}" if parent and d.name else (d.name or "repo")) + tags = widened + seen: dict[str, int] = {} + unique: list[str] = [] + for t in tags: + seen[t] = seen.get(t, 0) + 1 + unique.append(t if seen[t] == 1 else f"{t}-{seen[t]}") + return unique + + def prune_repo_from_graph(G: nx.Graph, repo_tag: str) -> int: """Remove all nodes tagged with repo_tag from G in-place. Returns count removed.""" to_remove = [n for n, d in G.nodes(data=True) if d.get("repo") == repo_tag] diff --git a/graphify/cache.py b/graphify/cache.py index bb3f9d593..7257aca72 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -7,6 +7,8 @@ import os import re import tempfile +import warnings +from collections.abc import Iterable from pathlib import Path # Output directory name — override with GRAPHIFY_OUT env var for worktrees or @@ -61,6 +63,99 @@ def _cleanup_stale_ast_entries(ast_base: Path, current_dir: Path) -> None: pass +# Semantic cache entries are LLM output, so they depend on the extraction prompt +# that produced them, not just on file contents. Keying purely on content means a +# release that changes the prompt keeps replaying entries from the older prompt on +# every unchanged file, silently mixing extraction vintages in one graph (#1939). +# Versioning them by package version (as the AST cache does) would re-bill LLM +# extraction on every patch release — the reason #1252 deliberately left them +# unversioned. Fingerprinting the prompt itself keeps both properties: entries +# survive releases that don't touch the prompt, and invalidate only when it +# actually changed. Entries live under cache/semantic/p{fingerprint}/ when the +# caller supplies its prompt; callers that don't keep the historical flat layout. +_PROMPT_FP_LEN = 12 + +# Count of pre-fingerprint (flat-layout) entries served this process, so +# check_semantic_cache can report N to the user (#1939). +_legacy_semantic_hits = 0 + +# Prompt-file fingerprints already computed, keyed by (path, size, mtime_ns) — +# the same stat signature the hash index uses. check_semantic_cache resolves the +# prompt once per FILE in the corpus, so without this a 500-doc run re-reads and +# re-hashes the same spec 500 times (and warns 500 times when it is unreadable). +_prompt_fp_cache: dict[tuple, str] = {} + + +def prompt_fingerprint(prompt: "str | Path") -> str: + """Return a short stable fingerprint of an extraction prompt. + + ``prompt`` is either the prompt text itself (the Python extraction path owns + its system prompt, :func:`graphify.llm._extraction_system`) or a Path to the + prompt file an agent loaded (the skill path's + ``references/extraction-spec.md``). + + Line endings and trailing whitespace are normalized before hashing: the same + spec file checked out with CRLF on Windows must not fingerprint differently + from the LF checkout that wrote the cache, or every Windows run would look + like a prompt change and re-bill extraction. + """ + if isinstance(prompt, Path): + text = prompt.read_text(encoding="utf-8", errors="replace") + else: + text = prompt + normalized = "\n".join( + line.rstrip() for line in text.replace("\r\n", "\n").replace("\r", "\n").split("\n") + ).strip() + return hashlib.sha256(normalized.encode()).hexdigest()[:_PROMPT_FP_LEN] + + +def _resolve_prompt_fp(prompt: "str | Path | None" = None, + prompt_file: "str | Path | None" = None) -> str | None: + """Fingerprint the caller's extraction prompt, or None when it supplied none. + + ``prompt`` is prompt TEXT; ``prompt_file`` is a path to a file CONTAINING the + prompt. They are separate parameters rather than one overloaded argument + because the skill-driven callers are markdown snippets an agent copies with a + path substituted in — passing that path as ``prompt`` would hash the path + string itself, yielding a fingerprint that is stable, plausible, and tracks + nothing about the prompt. A silent wrong fingerprint is the exact failure + class #1939 is about, so the two are not inferred from each other. + + Best-effort: an unreadable ``prompt_file`` falls back to the flat, unattributed + layout rather than failing the run — a cache is never worth aborting an + extraction over. It warns rather than falling back quietly, because that + fallback silently restores the very behavior this fixes, and the skill-side + caller substitutes this path by hand. + """ + memo_key = None + if prompt_file is not None: + prompt = Path(prompt_file) + try: + st = prompt.stat() + memo_key = (str(prompt), st.st_size, st.st_mtime_ns) + if memo_key in _prompt_fp_cache: + return _prompt_fp_cache[memo_key] + except OSError: + pass # unreadable — fall through to the warning below + if prompt is None: + return None + try: + fp = prompt_fingerprint(prompt) + if memo_key is not None: + _prompt_fp_cache[memo_key] = fp + return fp + except (OSError, UnicodeError) as exc: + warnings.warn( + f"could not read extraction prompt {str(prompt)!r} ({exc}); semantic cache " + "entries cannot be attributed to a prompt version and fall back to the " + "unversioned layout, so this run may replay entries from an older " + "extraction prompt (#1939).", + RuntimeWarning, + stacklevel=3, + ) + return None + + # A frontmatter delimiter is a whole line of exactly three dashes (optional # trailing whitespace). Substring checks like startswith("---") / # find("\n---") also match `----` thematic breaks and `--- text` prose, @@ -88,31 +183,89 @@ def _body_content(content: bytes) -> bytes: # size+mtime_ns are unchanged — same trade-off as make(1). # Correctness risks: `touch` causes a harmless extra re-hash; same-size edits # within NFS second-resolution mtime have a 1-second window (same as make). -# Use `graphify extract --force` to bypass when needed. +# `graphify extract --force` / `graphify update --force` (or GRAPHIFY_FORCE=1) +# skip the cache reads and re-dispatch everything when needed (#1894). _stat_index: dict[str, dict] = {} _stat_index_root: Path | None = None +# Key anchor for the ON-DISK index (#2199): the first caller's key-root, i.e. +# the corpus. Distinct from _stat_index_root, which is the cache-FILE location +# (cache_root, #1774) — the two differ under --out and must not be conflated. +_stat_index_anchor: Path | None = None _stat_index_dirty: bool = False +def _stat_key_to_relative(key: str, anchor: Path) -> str: + """Return ``key`` as a forward-slash relative path from ``anchor``. + + Local duplicate of :func:`graphify.detect._to_relative_for_storage` — + detect imports cache, so cache cannot import detect without a cycle + (and pulling detect in during the atexit flush would be fragile). + Out-of-anchor and already-relative keys pass through unchanged, and + ``..``-escaping relpaths are rejected (kept absolute), mirroring the + manifest's portability rules. + """ + p = Path(key) + if not p.is_absolute(): + return key + try: + rel = os.path.relpath(p, anchor) + except (ValueError, OSError): + return key # outside anchor (e.g. Windows cross-drive) + if rel == ".." or rel.startswith(".." + os.sep) or rel.startswith("../"): + return key # escaped anchor — keep absolute + return rel.replace(os.sep, "/") + + +def _stat_key_to_absolute(key: str, anchor: Path) -> str: + """Inverse of :func:`_stat_key_to_relative`. + + Re-anchor a stored relative key against ``anchor``. Already-absolute keys + (legacy indexes, out-of-anchor entries) pass through unchanged so an index + written by an older graphify remains readable. + """ + p = Path(key) + if p.is_absolute(): + return str(p) + return str(anchor / p) + + def _stat_index_file(root: Path) -> Path: _out = Path(_GRAPHIFY_OUT) base = _out if _out.is_absolute() else Path(root).resolve() / _out return base / "cache" / "stat-index.json" -def _ensure_stat_index(root: Path) -> None: - global _stat_index, _stat_index_root, _stat_index_dirty +def _ensure_stat_index(root: Path, cache_root: "Path | None" = None) -> None: + global _stat_index, _stat_index_root, _stat_index_anchor, _stat_index_dirty if _stat_index_root is not None: return - _stat_index_root = Path(root).resolve() + # _stat_index_root determines the cache FILE location, so honoring an + # explicit cache_root keeps detect()'s word-count cache under the requested + # --out dir instead of polluting the scanned corpus with a stray + # graphify-out/ (#1747). _stat_index_anchor is the separate KEY anchor: + # in-memory keys stay absolute, but the on-disk index stores in-anchor keys + # relative so a moved/cloned corpus still hits (#2199) — same load/save + # re-anchoring the detect manifest uses. + _stat_index_root = Path(cache_root if cache_root is not None else root).resolve() + _stat_index_anchor = Path(root).resolve() p = _stat_index_file(_stat_index_root) + _stat_index = {} if p.exists(): try: - _stat_index = json.loads(p.read_text(encoding="utf-8")) + raw = json.loads(p.read_text(encoding="utf-8")) + if isinstance(raw, dict): + for k, v in raw.items(): + if not isinstance(k, str): + continue + if Path(k).is_absolute(): + # Legacy/out-of-anchor key: pass through, but never + # clobber a re-anchored relative (new-format) entry + # that resolved to the same absolute path. + _stat_index.setdefault(k, v) + else: + _stat_index[_stat_key_to_absolute(k, _stat_index_anchor)] = v except (json.JSONDecodeError, OSError): _stat_index = {} - else: - _stat_index = {} atexit.register(_flush_stat_index) @@ -121,11 +274,26 @@ def _flush_stat_index() -> None: if not _stat_index_dirty or _stat_index_root is None: return p = _stat_index_file(_stat_index_root) + # Build the on-disk form (#2199): prune entries whose file is gone (the + # index otherwise grows without bound), then store in-anchor keys as + # forward-slash relative paths so the index survives a corpus move/clone. + # Out-of-anchor keys stay absolute (same rule as the detect manifest); a + # reader tells the formats apart by absoluteness, so no version marker is + # needed. In-memory keys are untouched — only the serialization changes. + on_disk: dict[str, dict] = {} + for k, v in _stat_index.items(): + try: + if not os.path.exists(k): + continue + except OSError: + continue + dk = _stat_key_to_relative(k, _stat_index_anchor) if _stat_index_anchor is not None else k + on_disk[dk] = v try: p.parent.mkdir(parents=True, exist_ok=True) fd, tmp = tempfile.mkstemp(dir=p.parent, prefix="stat-index.", suffix=".tmp") try: - os.write(fd, json.dumps(_stat_index, separators=(",", ":")).encode()) + os.write(fd, json.dumps(on_disk, separators=(",", ":")).encode()) os.close(fd) os.replace(tmp, p) except Exception: @@ -153,7 +321,7 @@ def _normalize_path(path: Path) -> Path: return Path(os.path.normcase(s)) -def file_hash(path: Path, root: Path = Path(".")) -> str: +def file_hash(path: Path, root: Path = Path("."), cache_root: "Path | None" = None) -> str: """SHA256 of file contents + path relative to root. Uses a stat-based fastpath (size + mtime_ns) to skip full reads when the @@ -173,17 +341,39 @@ def file_hash(path: Path, root: Path = Path(".")) -> str: if not p.is_file(): raise IsADirectoryError(f"file_hash requires a file, got: {p}") - _ensure_stat_index(root) - abs_key = str(p.resolve()) + # The stat index is a cache artifact, so it must follow the cache location + # (cache_root), not the key-anchor root — otherwise it leaves a stray + # graphify-out/cache/stat-index.json inside the analyzed source tree even when + # the AST cache itself is redirected to CWD (#1774 completion). + _ensure_stat_index(root, cache_root=cache_root) + resolved = p.resolve() + abs_key = str(resolved) + # The salt is the path component that enters the digest (relative to root, or + # the absolute-path fallback). The stat-index memo MUST be keyed by it too: + # the same file hashed under two different roots yields two different digests + # (this happens within one `--out` run), and a memo keyed only by absolute + # path served whichever was computed first — making file_hash order-dependent + # and poisoning the persisted stat-index across runs (#1989). Store one digest + # per salt so alternating roots don't force re-reads. + try: + salt = resolved.relative_to(Path(root).resolve()).as_posix().lower() + except ValueError: + salt = resolved.as_posix().lower() + st: "os.stat_result | None" = None try: st = p.stat() entry = _stat_index.get(abs_key) - if (entry - and entry.get("hash") is not None # word-count-only entries carry no hash + if (isinstance(entry, dict) and entry.get("size") == st.st_size and entry.get("mtime_ns") == st.st_mtime_ns): - return entry["hash"] + hashes = entry.get("hashes") + if isinstance(hashes, dict): + cached = hashes.get(salt) + if isinstance(cached, str): + return cached + # Legacy single-digest entries ("hash") don't record which salt + # produced them, so they are never trusted (#1989) — recompute once. except OSError: pass @@ -192,27 +382,29 @@ def file_hash(path: Path, root: Path = Path(".")) -> str: h = hashlib.sha256() h.update(content) h.update(b"\x00") - try: - rel = p.resolve().relative_to(Path(root).resolve()) - h.update(rel.as_posix().lower().encode()) - except ValueError: - h.update(p.resolve().as_posix().lower().encode()) + h.update(salt.encode()) digest = h.hexdigest() if st is not None: entry = _stat_index.get(abs_key) - if (entry is not None + if (isinstance(entry, dict) and entry.get("size") == st.st_size and entry.get("mtime_ns") == st.st_mtime_ns): - entry["hash"] = digest # preserve a co-located word_count + hashes = entry.get("hashes") + if not isinstance(hashes, dict): + hashes = {} + entry["hashes"] = hashes + hashes[salt] = digest # preserve a co-located word_count / other salts + entry.pop("hash", None) # retire the un-salted legacy digest else: - _stat_index[abs_key] = {"size": st.st_size, "mtime_ns": st.st_mtime_ns, "hash": digest} + _stat_index[abs_key] = {"size": st.st_size, "mtime_ns": st.st_mtime_ns, + "hashes": {salt: digest}} _stat_index_dirty = True return digest -def cached_word_count(path: Path, root: Path, compute) -> int: +def cached_word_count(path: Path, root: Path, compute, cache_root: "Path | None" = None) -> int: """Word count with the same (size, mtime_ns) stat-fastpath cache as :func:`file_hash`, persisted in the shared stat index. @@ -228,7 +420,7 @@ def cached_word_count(path: Path, root: Path, compute) -> int: global _stat_index_dirty p = _normalize_path(Path(path)) root = _normalize_path(Path(root)) - _ensure_stat_index(root) + _ensure_stat_index(root, cache_root=cache_root) abs_key = str(p.resolve()) st: "os.stat_result | None" = None try: @@ -276,7 +468,11 @@ def _relativize_source_files_in(payload: dict, root: Path) -> None: root_resolved = Path(root).resolve() except OSError: return - for bucket in ("nodes", "edges", "hyperedges"): + # raw_calls (#: Pascal/Delphi cross-file inherited-call resolution) carries + # source_file the same way nodes/edges/hyperedges do, so it needs the same + # portable-path treatment for cache entries to round-trip correctly across + # machines/checkout directories. + for bucket in ("nodes", "edges", "hyperedges", "raw_calls"): for item in payload.get(bucket, []): if not isinstance(item, dict): continue @@ -295,6 +491,29 @@ def _relativize_source_files_in(payload: dict, root: Path) -> None: item["source_file"] = rel.replace(os.sep, "/") +def _normalize_source_file_value(src: "str | Path", root_resolved: Path) -> str: + """Return ``src`` in portable form: backslashes flipped to forward slashes, + then relativized against ``root_resolved`` when the path is in-root. + + Windows ``detect()`` emits absolute backslash paths, and a semantic + fragment carrying one verbatim used to be persisted as-is — poisoning later + ``graphify update`` runs with a machine-specific ``source_file`` (#2197). + Out-of-root absolute paths pass through (slash-normalized only), same + in/out rule and ``..``-rejection as :func:`_relativize_source_files_in`. + """ + s = str(src).replace("\\", "/") + p = Path(s) + if not p.is_absolute(): + return s + try: + rel = os.path.relpath(p, root_resolved) + except (ValueError, OSError): + return s # out-of-root (e.g. Windows cross-drive) + if rel == ".." or rel.startswith(".." + os.sep) or rel.startswith("../"): + return s # escaped root — keep absolute + return rel.replace(os.sep, "/") + + def _absolutize_source_files_in(payload: dict, root: Path) -> None: """Inverse of :func:`_relativize_source_files_in`. @@ -307,7 +526,7 @@ def _absolutize_source_files_in(payload: dict, root: Path) -> None: root_resolved = Path(root).resolve() except OSError: return - for bucket in ("nodes", "edges", "hyperedges"): + for bucket in ("nodes", "edges", "hyperedges", "raw_calls"): for item in payload.get(bucket, []): if not isinstance(item, dict): continue @@ -323,16 +542,24 @@ def _absolutize_source_files_in(payload: dict, root: Path) -> None: continue -def cache_dir(root: Path = Path("."), kind: str = "ast") -> Path: +def cache_dir(root: Path = Path("."), kind: str = "ast", + prompt_fp: str | None = None) -> Path: """Returns the cache directory for ``kind`` - creates it if needed. - kind is "ast" or "semantic". Separate subdirectories prevent semantic cache + kind is "ast", "semantic", or a mode-namespaced semantic kind such as + "semantic-deep" (#1894). Separate subdirectories prevent semantic cache entries from overwriting AST cache entries for the same source_file (#582). AST entries live in graphify-out/cache/ast/v{version}/ — namespaced by graphify version because they depend on extractor code, not just file - contents. Semantic entries live unversioned in graphify-out/cache/semantic/ - (re-extraction costs LLM calls). + contents. Semantic entries are still NOT version-namespaced (re-extraction + costs LLM calls, #1252): they live in graphify-out/cache/semantic/, with + deep-mode entries beside them in graphify-out/cache/semantic-deep/. + + ``prompt_fp`` (semantic kinds only) adds a p{fingerprint}/ subdirectory so + entries are attributed to the extraction prompt that produced them (#1939). + Omitting it yields the historical flat layout, where entries of unknown + vintage live. """ _out = Path(_GRAPHIFY_OUT) base = _out if _out.is_absolute() else Path(root).resolve() / _out @@ -340,33 +567,78 @@ def cache_dir(root: Path = Path("."), kind: str = "ast") -> Path: if kind == "ast": d = d / f"v{_EXTRACTOR_VERSION}" _cleanup_stale_ast_entries(d.parent, d) + elif prompt_fp: + d = d / f"p{prompt_fp}" d.mkdir(parents=True, exist_ok=True) return d -def load_cached(path: Path, root: Path = Path("."), kind: str = "ast") -> dict | None: +def load_cached(path: Path, root: Path = Path("."), kind: str = "ast", + cache_root: Path | None = None, prompt: "str | Path | None" = None, + prompt_file: "str | Path | None" = None, + allow_legacy: bool = True, + allow_partial: bool = False) -> dict | None: """Return cached extraction for this file if hash matches, else None. Cache key: SHA256 of file contents. Cache value: stored as graphify-out/cache/{kind}/{hash}.json (AST entries under the per-version subdirectory, see :func:`cache_dir`). + ``root`` anchors the content-hash key and source_file relativization (it + must stay the inferred common parent so keys remain portable). ``cache_root`` + decouples *where* the cache directory lives from that anchor — the cache is + an output and must not land inside a read-only/analyzed source tree (#1774). + When ``cache_root`` is None the location falls back to ``root`` (unchanged + behavior for existing callers). + AST entries written by other graphify versions — including the legacy flat cache/ layout (pre-0.5.3) and the unversioned cache/ast/ layout — are deliberately not consulted: they were produced by a different extractor and may be stale. + + ``prompt`` (semantic kinds) is the extraction prompt — text, or a Path to + the prompt file — that the caller is about to extract with. It selects the + p{fingerprint}/ namespace, so an entry produced by a different prompt is a + miss rather than a silent stale hit (#1939). When it is given and the + fingerprinted namespace misses, ``allow_legacy`` (default True) falls back + to a flat-layout entry: those predate fingerprinting, so their vintage is + unknowable — they are served rather than re-billed, and the hit is counted + so :func:`check_semantic_cache` can report N to the user. Callers that must + not mix vintages within one entry (see :func:`save_semantic_cache`'s + ``merge_existing``) pass allow_legacy=False. Returns None if no cache entry or file has changed. """ + global _legacy_semantic_hits + location = cache_root if cache_root is not None else root try: - h = file_hash(path, root) + h = file_hash(path, root, cache_root=cache_root) except OSError: return None - entry = cache_dir(root, kind) / f"{h}.json" + prompt_fp = _resolve_prompt_fp(prompt, prompt_file) + entry = cache_dir(location, kind, prompt_fp) / f"{h}.json" + legacy_hit = False + if prompt_fp and not entry.exists() and allow_legacy: + legacy = cache_dir(location, kind) / f"{h}.json" + if legacy.exists(): + entry, legacy_hit = legacy, True if entry.exists(): try: result = json.loads(entry.read_text(encoding="utf-8")) except (json.JSONDecodeError, OSError): return None + # A ``partial`` entry was produced from a truncated LLM response and + # covers only part of the file's symbols. Serving it as authoritative + # would return the incomplete node set forever until the file is + # re-extracted. Treat it as a cache MISS (the normal read path) so the + # file is re-dispatched and retried. Self-heals: a later complete + # extraction overwrites the same content-hash key with a non-partial + # entry. ``allow_partial`` is the one exception — the merge_existing + # checkpoint peeks at a partial prev so it can accumulate a file's slices + # across chunks without losing the truncated one (it stays partial). + if not allow_partial and isinstance(result, dict) and result.get("partial"): + return None + if legacy_hit: + _legacy_semantic_hits += 1 # Re-anchor relative source_file fields so callers see the same # absolute-path shape that a fresh in-process extraction produces # (#777). Legacy entries with absolute source_file pass through. @@ -376,12 +648,25 @@ def load_cached(path: Path, root: Path = Path("."), kind: str = "ast") -> dict | return None -def save_cached(path: Path, result: dict, root: Path = Path("."), kind: str = "ast") -> None: +def save_cached(path: Path, result: dict, root: Path = Path("."), kind: str = "ast", + cache_root: Path | None = None, prompt: "str | Path | None" = None, + prompt_file: "str | Path | None" = None) -> None: """Save extraction result for this file. Stores as graphify-out/cache/{kind}/{hash}.json where hash = SHA256 of current file contents. result should be a dict with 'nodes' and 'edges' lists. + ``root`` anchors the content-hash key and source_file relativization; + ``cache_root`` (when given) is where the cache directory is written, decoupled + from ``root`` so the cache never lands inside the analyzed source tree (#1774). + + ``prompt`` (semantic kinds) is the extraction prompt that produced ``result`` + — text, or a Path to the prompt file. It stamps the entry into the + p{fingerprint}/ namespace so a later run under a different prompt does not + replay it (#1939). Writes always land in the fingerprinted namespace when a + prompt is given: an entry of known vintage is never written back into the + flat unknown-vintage layout. + No-ops if `path` is not a regular file. Subagent-produced semantic fragments occasionally carry a directory path in `source_file`; skipping them prevents IsADirectoryError from aborting the whole batch. @@ -400,12 +685,13 @@ def save_cached(path: Path, result: dict, root: Path = Path("."), kind: str = "a # source_file field's original absolute form. Mutating the input here would # silently break those remaps on the first extraction pass. on_disk = result - if isinstance(result, dict) and any(result.get(k) for k in ("nodes", "edges", "hyperedges")): + if isinstance(result, dict) and any(result.get(k) for k in ("nodes", "edges", "hyperedges", "raw_calls")): import copy as _copy on_disk = _copy.deepcopy(result) _relativize_source_files_in(on_disk, root) - h = file_hash(p, root) - target_dir = cache_dir(root, kind) + h = file_hash(p, root, cache_root=cache_root) + location = cache_root if cache_root is not None else root + target_dir = cache_dir(location, kind, _resolve_prompt_fp(prompt, prompt_file)) entry = target_dir / f"{h}.json" fd, tmp_path = tempfile.mkstemp(dir=target_dir, prefix=f"{h}.", suffix=".tmp") try: @@ -438,26 +724,32 @@ def cached_files(root: Path = Path(".")) -> set[str]: # Legacy flat entries if base.is_dir(): hashes.update(p.stem for p in base.glob("*.json")) - # Namespaced entries (ast/ recursively, covering per-version subdirs) - for kind, pattern in (("ast", "**/*.json"), ("semantic", "*.json")): + # Namespaced entries, all globbed recursively: ast/ has per-version subdirs, + # semantic-deep/ holds --mode deep entries (#1894), and both semantic kinds + # have per-prompt-fingerprint subdirs alongside pre-fingerprint flat entries + # (#1939). + for kind in ("ast", "semantic", "semantic-deep"): d = base / kind if d.is_dir(): - hashes.update(p.stem for p in d.glob(pattern)) + hashes.update(p.stem for p in d.glob("**/*.json")) return hashes def clear_cache(root: Path = Path(".")) -> None: - """Delete all cache entries (ast/, semantic/, and legacy flat entries).""" + """Delete all cache entries (ast/, semantic/, semantic-deep/, and legacy + flat entries).""" base = Path(root).resolve() / _GRAPHIFY_OUT / "cache" # Legacy flat entries if base.is_dir(): for f in base.glob("*.json"): f.unlink() - # Namespaced entries (ast/ recursively, covering per-version subdirs) - for kind, pattern in (("ast", "**/*.json"), ("semantic", "*.json")): + # Namespaced entries, all globbed recursively: ast/ has per-version subdirs, + # semantic-deep/ holds --mode deep entries (#1894), and both semantic kinds + # have per-prompt-fingerprint subdirs (#1939). + for kind in ("ast", "semantic", "semantic-deep"): d = base / kind if d.is_dir(): - for f in d.glob(pattern): + for f in d.glob("**/*.json"): f.unlink() @@ -471,11 +763,26 @@ def prune_semantic_cache(root: Path, live_hashes: set[str]) -> int: AST version-cleanup, so every content change or file deletion leaves a permanent orphan entry that accumulates unbounded. - This sweeps ``cache/semantic/*.json`` and deletes any entry whose stem (the - content hash) is not in ``live_hashes`` — the hashes of the current live - document set. ``*.tmp`` atomic-write temporaries are skipped, and only this - directory is touched (never ``cache/ast/**`` or anything else). The - unversioned design is preserved: we prune by liveness, not by version. + This sweeps ``cache/semantic/*.json`` AND ``cache/semantic-deep/*.json`` + (the ``--mode deep`` namespace, #1894) and deletes any entry whose stem + (the content hash) is not in ``live_hashes`` — the hashes of the current + live document set. Both namespaces are pruned against the SAME live set: + liveness is content-based and mode-independent, so a hash that is live for + one namespace is live for both. Skipping the deep namespace would re-grow + the unbounded-orphan problem this function fixed (#1527). ``*.tmp`` + atomic-write temporaries are skipped, and only these directories are + touched (never ``cache/ast/**`` or anything else). The unversioned design + is preserved: we prune by liveness, not by version. + + The sweep recurses into the per-prompt-fingerprint subdirs (#1939) for the + same reason it covers the deep namespace: a glob that stopped at the top + level would leave every fingerprinted entry permanently unprunable. Entries + under a fingerprint other than the current one are pruned by liveness only, + never swept wholesale the way :func:`_cleanup_stale_ast_entries` sweeps old + AST versions — two hosts with different prompts (verbose vs compact + extraction-spec) can share one graphify-out/, and a wholesale sweep would + have each run delete the other's entries and re-bill extraction on every + alternation. Liveness keeps the total bounded by live docs × prompts seen. Best-effort, mirroring :func:`_cleanup_stale_ast_entries`: each unlink is wrapped in ``try/except OSError`` and a failure is ignored. The worst-case @@ -484,40 +791,72 @@ def prune_semantic_cache(root: Path, live_hashes: set[str]) -> int: """ _out = Path(_GRAPHIFY_OUT) base = _out if _out.is_absolute() else Path(root).resolve() / _out - semantic_dir = base / "cache" / "semantic" - if not semantic_dir.is_dir(): - return 0 pruned = 0 - for entry in semantic_dir.glob("*.json"): - if entry.stem in live_hashes: + for kind in ("semantic", "semantic-deep"): + semantic_dir = base / "cache" / kind + if not semantic_dir.is_dir(): continue - try: - entry.unlink() - pruned += 1 - except OSError: - pass + for entry in semantic_dir.glob("**/*.json"): + if entry.stem in live_hashes: + continue + try: + entry.unlink() + pruned += 1 + except OSError: + pass return pruned def check_semantic_cache( files: list[str], root: Path = Path("."), + mode: str | None = None, + prompt: "str | Path | None" = None, + prompt_file: "str | Path | None" = None, + cache_root: "Path | None" = None, ) -> tuple[list[dict], list[dict], list[dict], list[str]]: """Check semantic extraction cache for a list of absolute file paths. Returns (cached_nodes, cached_edges, cached_hyperedges, uncached_files). Uncached files need Claude extraction; cached files are merged directly. + + ``mode`` selects the cache namespace: ``None`` (the default) reads + ``cache/semantic/`` — byte-identical to the historical behavior, so + existing callers that omit it (including older installed skill flows) + are unaffected. A non-None mode (e.g. ``"deep"``) reads + ``cache/semantic-{mode}/`` instead, so deep-mode results never shadow + (or get shadowed by) standard-mode entries for the same content (#1894). + + ``prompt`` is the extraction prompt this run will use for the uncached + files — the prompt text (Python path) or a Path to the prompt file the + agent loaded (skill path, ``references/extraction-spec.md``). Supplying it + restricts hits to entries produced by that same prompt, so an upgrade that + changed the prompt re-extracts instead of replaying the older vintage + (#1939). Entries written before fingerprinting existed still hit — their + vintage is unknowable and dropping them would re-bill a whole corpus — but + a warning reports how many were served. Omitting ``prompt`` keeps the + historical behavior for existing callers. + + ``cache_root`` decouples *where* the cache is read from the key-anchor + ``root``, mirroring :func:`load_cached` and :func:`save_semantic_cache` + (#1774 / #1990). With ``--out``, pass the corpus as ``root`` (so content-hash + keys and relative-path resolution stay anchored to the source tree) and the + output directory as ``cache_root``. Omitting it keeps ``root`` for both. """ + global _legacy_semantic_hits + kind = "semantic" if mode is None else f"semantic-{mode}" cached_nodes: list[dict] = [] cached_edges: list[dict] = [] cached_hyperedges: list[dict] = [] uncached: list[str] = [] + legacy_before = _legacy_semantic_hits for fpath in files: p = Path(fpath) if not p.is_absolute(): p = Path(root) / p - result = load_cached(p, root, kind="semantic") + result = load_cached(p, root, kind=kind, cache_root=cache_root, + prompt=prompt, prompt_file=prompt_file) if result is not None: cached_nodes.extend(result.get("nodes", [])) cached_edges.extend(result.get("edges", [])) @@ -525,44 +864,289 @@ def check_semantic_cache( else: uncached.append(fpath) + legacy = _legacy_semantic_hits - legacy_before + if legacy: + warnings.warn( + f"{legacy} semantic cache entr{'y' if legacy == 1 else 'ies'} predate " + "extraction-prompt fingerprinting and were written by an unknown prompt " + "version; they were replayed as-is, so this graph may mix extraction " + "vintages. Re-run with --force (or GRAPHIFY_FORCE=1) to re-extract them " + "with the current prompt (#1939).", + RuntimeWarning, + stacklevel=2, + ) + return cached_nodes, cached_edges, cached_hyperedges, uncached +def _group_has_partial_marker(group: dict) -> bool: + """True if any node/edge/hyperedge in a per-file group carries the internal + ``_partial`` truncation marker set by the adaptive-retry give-up sites. + + The marker rides the item dicts up through every chunk merge, so it reaches + ``save_semantic_cache`` on BOTH the incremental checkpoint path (llm.py) and + the final authoritative save (cli.py) without either caller having to thread + an extra argument — the final save would otherwise overwrite a checkpoint's + ``partial`` flag with a clean-looking entry. + """ + for bucket in ("nodes", "edges", "hyperedges"): + for item in group.get(bucket, []): + if isinstance(item, dict) and item.get("_partial"): + return True + return False + + def save_semantic_cache( nodes: list[dict], edges: list[dict], hyperedges: list[dict] | None = None, root: Path = Path("."), + merge_existing: bool = False, + allowed_source_files: Iterable[str | Path] | None = None, + mode: str | None = None, + prompt: "str | Path | None" = None, + prompt_file: "str | Path | None" = None, + partial_source_files: Iterable[str | Path] | None = None, + cache_root: "Path | None" = None, ) -> int: """Save semantic extraction results to cache, keyed by source_file. Groups nodes and edges by source_file, then saves one cache entry per file under cache/semantic/ (separate from AST entries in cache/ast/) to prevent hash-key collisions (#582). + + ``mode`` selects the cache namespace, mirroring + :func:`check_semantic_cache`: ``None`` (the default) writes + ``cache/semantic/`` — byte-identical to the historical behavior for + existing callers that omit it — while a non-None mode (e.g. ``"deep"``) + writes ``cache/semantic-{mode}/`` so richer deep-mode results never + overwrite standard-mode entries and vice versa (#1894). + + When ``merge_existing`` is True, any already-cached entry for a file is + unioned with the new results before saving instead of being overwritten. + This lets callers checkpoint incrementally (e.g. once per chunk) without + dropping a prior slice of a large file that was split across chunks. + + When ``allowed_source_files`` is provided, only those files may be used as + cache-write keys. Semantic nodes can legitimately mention another corpus + file, but a model must not be able to replace that file's complete cache + entry unless the file was part of the current extraction batch (#1757). + + When ``partial_source_files`` is provided, entries for those files are + stamped ``partial: True`` — the extraction was truncated, so the entry is + incomplete and :func:`load_cached` must treat it as a miss. Partial-ness is + ALSO detected intrinsically from a ``_partial`` marker on any grouped item, + so the flag survives even when a caller (e.g. cli.py's final save) does not + pass ``partial_source_files``. + + ``prompt`` is the extraction prompt that produced these results — text, or + a Path to the prompt file. It stamps entries into the p{fingerprint}/ + namespace so a later run under a different prompt re-extracts rather than + replaying them (#1939). Pass the same prompt here as to + :func:`check_semantic_cache`, or the write lands in a namespace the next + read won't consult. + + ``cache_root`` decouples *where* the cache directory is written from the + source-key anchor ``root`` — mirroring the same split that :func:`load_cached` + and :func:`save_cached` already expose (#1774). When given, cache files land + under ``cache_root`` while ``source_file`` paths are still resolved and + relativized against ``root``. When omitted, ``root`` is used for both + purposes (unchanged behaviour for existing callers). This fixes checkpoints + and the final save going to the corpus tree instead of ``--out`` (#1990, + #1991). + Returns the number of files cached. """ from collections import defaultdict + kind = "semantic" if mode is None else f"semantic-{mode}" + root_path = Path(root).resolve() + + def _normalized(item: dict) -> dict: + """Copy of ``item`` with a portable ``source_file`` (#2197). + + Normalizing BEFORE grouping means both the group key and the persisted + item carry the relative forward-slash form, so a fragment whose + source_file arrived absolute (Windows detect() output) can never be + cached verbatim. A shallow copy keeps the caller's dicts untouched — + downstream steps may still rely on the original absolute shape (same + reasoning as :func:`save_cached`'s on-disk deepcopy). + """ + src = item.get("source_file") + if not src: + return item + norm = _normalize_source_file_value(src, root_path) + if norm != src: + item = {**item, "source_file": norm} + return item + by_file: dict[str, dict] = defaultdict(lambda: {"nodes": [], "edges": [], "hyperedges": []}) for n in nodes: + n = _normalized(n) src = n.get("source_file", "") if src: by_file[src]["nodes"].append(n) for e in edges: + e = _normalized(e) src = e.get("source_file", "") if src: by_file[src]["edges"].append(e) for h in (hyperedges or []): + h = _normalized(h) src = h.get("source_file", "") if src: by_file[src]["hyperedges"].append(h) + def resolved_source_path(value: str | Path) -> Path: + path = Path(value) + if not path.is_absolute(): + path = root_path / path + try: + return path.resolve() + except (OSError, RuntimeError): + # Keep the cache write best-effort for inaccessible paths or a + # symlink loop emitted by an untrusted semantic result. + return Path(os.path.abspath(path)) + + allowed_paths = None + if allowed_source_files is not None: + allowed_paths = {resolved_source_path(path) for path in allowed_source_files} + + partial_paths = None + if partial_source_files is not None: + partial_paths = {resolved_source_path(path) for path in partial_source_files} + # A chunk that truncated to an EMPTY parse contributes no grouped items, + # so its file is absent from by_file and the write loop below would never + # stamp it partial — leaving a prior clean slice looking complete (#1950 + # empty-parse gap). Seed an empty group for each named partial file that + # isn't already present, so the loop merges its existing entry and stamps + # it partial. Keyed by the resolved path (deduped against present groups). + _present = {resolved_source_path(k) for k in by_file} + for _pp in partial_paths: + if _pp not in _present: + by_file[str(_pp)] # defaultdict: create an empty {nodes,edges,hyperedges} + + def group_skipped(fpath: str) -> bool: + """Mirror the write-loop skip condition for one source_file group.""" + p = resolved_source_path(fpath) + return not p.is_file() or (allowed_paths is not None and p not in allowed_paths) + + # Dangling-reference pruning (#1916). A node group is skipped by the write + # loop below when its source_file is not a real file (ghost path) or is + # out-of-scope per the #1757 guard — but an edge/hyperedge in an ALLOWED + # group that references a node id from a skipped group used to be written + # verbatim, so on replay (check_semantic_cache) it dangled forever (the + # #1895 merged-result filter runs AFTER this checkpoint write and is + # bypassed entirely on replay). Compute the node ids that will be skipped + # and drop any to-be-written edge whose endpoint — or hyperedge whose + # member (whole-hyperedge drop, mirroring #1895) — references one. Gated + # on allowed_source_files so unscoped callers stay byte-identical. + if allowed_paths is not None: + skipped_ids: set = set() + written_ids: set = set() + for fpath, result in by_file.items(): + target = skipped_ids if group_skipped(fpath) else written_ids + for n in result["nodes"]: + nid = n.get("id") + if nid is None: + continue + try: + hash(nid) + except TypeError: + continue + target.add(nid) + # A duplicate-attribution node (defined in a skipped AND a written + # group) still reaches the cache — don't over-prune references to it. + skipped_ids -= written_ids + if skipped_ids: + + def edge_dangles(e: dict) -> bool: + try: + return e.get("source") in skipped_ids or e.get("target") in skipped_ids + except TypeError: + # Non-hashable endpoint from an untrusted result; leave it + # to build-time validation rather than fail the save. + return False + + def hyperedge_dangles(h: dict) -> bool: + try: + return bool(skipped_ids & set(h.get("nodes") or [])) + except TypeError: + return False + + for fpath, result in by_file.items(): + if group_skipped(fpath): + continue + result["edges"] = [e for e in result["edges"] if not edge_dangles(e)] + result["hyperedges"] = [ + h for h in result["hyperedges"] if not hyperedge_dangles(h) + ] + saved = 0 + skipped_not_file = 0 for fpath, result in by_file.items(): - p = Path(fpath) - if not p.is_absolute(): - p = Path(root) / p + p = resolved_source_path(fpath) if p.is_file(): - save_cached(p, result, root, kind="semantic") + if allowed_paths is not None and p not in allowed_paths: + warnings.warn( + "semantic cache skipped out-of-scope source_file " + f"{fpath!r}; the file was not dispatched for extraction", + RuntimeWarning, + stacklevel=2, + ) + continue + if merge_existing: + # allow_legacy=False: merging a pre-fingerprint entry into this + # write would fuse two prompt vintages inside a single entry and + # then stamp the result as current-vintage — the exact mixing + # #1939 is about, made unfixable because the entry now claims a + # prompt that only produced half of it. + # allow_partial=True: a file split into slices across chunks + # accumulates here; if an earlier slice truncated, keep its nodes + # in the union AND let the entry stay partial (the _partial + # markers ride through, so is_partial below re-detects it) rather + # than a later clean slice silently replacing it and promoting the + # half-file to complete. + prev = load_cached(p, root, kind=kind, cache_root=cache_root, + prompt=prompt, prompt_file=prompt_file, + allow_legacy=False, allow_partial=True) + _prev_partial = bool(prev.get("partial")) if prev else False + if prev: + result = { + "nodes": (prev.get("nodes", []) or []) + result["nodes"], + "edges": (prev.get("edges", []) or []) + result["edges"], + "hyperedges": (prev.get("hyperedges", []) or []) + result["hyperedges"], + } + else: + _prev_partial = False + # A file is partial if the caller named it, any of its grouped items + # carries the intrinsic ``_partial`` marker, OR the entry it merged + # onto was already partial (an empty-parse truncation leaves a + # ``partial: True`` entry with no item markers, so a later clean slice + # merging over it must NOT silently promote the half-file to complete + # — #1950). Copy so the caller's dict is never mutated. A genuine + # complete re-extraction (merge_existing=False) overwrites the + # content-hash key with a non-partial entry that then serves normally. + is_partial = ( + (partial_paths is not None and p in partial_paths) + or _group_has_partial_marker(result) + or _prev_partial + ) + if is_partial: + result = {**result, "partial": True} + save_cached(p, result, root, kind=kind, cache_root=cache_root, + prompt=prompt, prompt_file=prompt_file) saved += 1 + else: + skipped_not_file += 1 + if skipped_not_file and skipped_not_file == len(by_file): + warnings.warn( + f"save_semantic_cache: all {skipped_not_file} source_file group(s) were " + "skipped because their paths do not resolve to real files. This usually " + "means ``root`` is anchored to the wrong directory (e.g. the --out " + "directory instead of the corpus root). Pass the corpus directory as " + "``root`` and the output directory as ``cache_root`` (#1991).", + RuntimeWarning, + stacklevel=2, + ) return saved diff --git a/graphify/callflow_html.py b/graphify/callflow_html.py index 181e7493b..dabb2997c 100644 --- a/graphify/callflow_html.py +++ b/graphify/callflow_html.py @@ -227,12 +227,13 @@ def _node_link_payload(data: dict) -> tuple[list, list] | None: return None try: - from networkx.readwrite import json_graph + # Shared loader normalizes the raw writer's "edges" key to "links" + # before parsing; without it an edges-keyed payload raised + # KeyError: 'links' and this function silently returned None even + # though the shape check above accepts "edges" (#2212). + from graphify.paths import load_node_link_graph - try: - graph = json_graph.node_link_graph(data, edges="links") - except TypeError: - graph = json_graph.node_link_graph(data) + graph = load_node_link_graph(data) except Exception: return None diff --git a/graphify/cargo_introspect.py b/graphify/cargo_introspect.py index 3bda96126..fa03ed309 100644 --- a/graphify/cargo_introspect.py +++ b/graphify/cargo_introspect.py @@ -78,8 +78,19 @@ def introspect_cargo(root: str | Path) -> dict[str, Any]: if not isinstance(dependencies, dict): continue source_file = manifest.relative_to(root_path).as_posix() - for dependency_name in sorted(dependencies): - target = crates.get(dependency_name) + for dep_key, spec in sorted(dependencies.items()): + # Cargo lets a dep table entry rename the crate via `package = "..."`: + # db = { path = "../storage", package = "internal-storage" } + # The key `db` is the name used in `use db::…;`; the actual crate + # published under `[package].name = "internal-storage"` is what + # `crates` is keyed by. Without honoring `package`, every renamed + # workspace-internal dep silently drops its edge (#1858). + real_name = dep_key + if isinstance(spec, dict): + pkg_override = spec.get("package") + if isinstance(pkg_override, str) and pkg_override: + real_name = pkg_override + target = crates.get(real_name) if target is None: continue edges.append( diff --git a/graphify/cli.py b/graphify/cli.py new file mode 100644 index 000000000..bd8f12bca --- /dev/null +++ b/graphify/cli.py @@ -0,0 +1,3833 @@ +"""graphify command dispatch — every non-install subcommand. + +Extracted verbatim from __main__.main(); __main__ now calls dispatch_command(cmd) +after the install/platform dispatch. Kept out of __main__ to shrink the CLI entry +module. The path-redirect (`graphify ` -> extract) re-enters via a lazy +import of main to avoid a cli<->__main__ import cycle. +""" +from __future__ import annotations +import json +import os +import re +import sys +import time +from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT +from pathlib import Path + + +_SEARCH_NUDGE = json.dumps({ + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "additionalContext": ( + 'MANDATORY: graphify-out/graph.json exists. You MUST run ' + '`graphify query ""` before grepping raw files. Only grep ' + 'after graphify has oriented you, or to modify/debug specific lines.' + ), + } +}, ensure_ascii=False, separators=(",", ":")) + "\n" +_READ_NUDGE = json.dumps({ + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "additionalContext": ( + 'MANDATORY: graphify-out/graph.json exists. You MUST run graphify ' + 'before reading source files. Use: `graphify query ""` ' + '(scoped subgraph), `graphify explain ""`, or ' + '`graphify path "" ""`. Only read raw files after graphify has ' + 'oriented you, or to modify/debug specific lines. This rule applies to ' + 'subagents too — include it in every subagent prompt involving code ' + 'exploration.' + ), + } +}, ensure_ascii=False, separators=(",", ":")) + "\n" +_READ_NUDGE_STALE = json.dumps({ + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "additionalContext": ( + 'graphify-out/graph.json exists but may be STALE for this file (the file ' + 'changed after the last build). Prefer `graphify query ""` for ' + 'orientation, and run `graphify update` to refresh the graph. Reading the ' + 'file directly is fine.' + ), + } +}, ensure_ascii=False, separators=(",", ":")) + "\n" +# Strict-mode block (opt-in). Claude Code PreToolUse honors +# hookSpecificOutput.permissionDecision == "deny" and shows permissionDecisionReason +# to the model. Fires at most once per session (see _mark_session_denied) so it can +# never strand an agent: the very next read proceeds with the soft nudge. +_READ_DENY = json.dumps({ + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": ( + 'graphify strict mode: this project has a fresh knowledge graph that covers ' + 'this file. Run `graphify query ""` (or `graphify explain` / ' + '`graphify path`) FIRST to orient yourself, then re-issue this Read — it ' + 'will be allowed. This block fires at most once per session; reading raw ' + 'files to modify or debug specific lines is fine after one query. Apply the ' + 'same rule in any subagent prompt that explores code.' + ), + } +}, ensure_ascii=False, separators=(",", ":")) + "\n" +_HOOK_SOURCE_EXTS = ( + '.py', '.js', '.cjs', '.ts', '.tsx', '.jsx', '.astro', '.vue', '.svelte', '.go', + '.rs', '.java', '.rb', '.c', '.h', '.cpp', '.hpp', '.cc', '.cs', '.kt', + '.swift', '.php', '.scala', '.lua', '.sh', '.md', '.rst', '.txt', '.mdx', +) +_GEMINI_NUDGE_TEXT = ( + 'graphify: knowledge graph at graphify-out/. For focused questions, run ' + '`graphify query ""` (scoped subgraph, usually much smaller than ' + 'GRAPH_REPORT.md) instead of grepping raw files. Read GRAPH_REPORT.md only ' + 'for broad architecture context.' +) + + +def _default_graph_path() -> str: + return str(Path(_GRAPHIFY_OUT) / "graph.json") + + +def _stamped_manifest_files( + files_by_type: dict[str, list[str]], + sem_result: dict, + root: Path, + partial_source_files: "set[str] | None" = None, +) -> dict[str, list[str]]: + """Manifest-safe files dict: only stamp semantic files that actually + produced output (cache hit or fresh extraction). Files whose chunk failed + have no source_file entry in sem_result — leaving their semantic_hash + empty so detect_incremental re-queues them (#933). + + A file in ``partial_source_files`` DID produce output this run, but only a + truncated fragment of it, so it is excluded from stamping too — otherwise + detect_incremental would see it "done" and never re-dispatch it, leaving the + incomplete node set live forever on the warm-incremental path. Same #933 + mechanism: leave it unstamped and it is re-queued next run. + + Both sides of the membership test are resolved against the scan ``root`` + before comparing (#1897): node/edge/hyperedge ``source_file`` values are + root-relative on a fresh extraction while ``files_by_type`` entries are + absolute (from detect()), so a raw string comparison never matched and + every freshly-extracted semantic doc was dropped from the manifest. + Mirrors the #1890 path normalization in graphify.llm. + + Hyperedges are counted as output (#1920): a chunk whose only result for a + document is a hyperedge (3+ nodes sharing a concept) is valid output that + the semantic cache persists per-``source_file`` — omitting it here left the + doc unstamped, so detect_incremental re-queued it on every run. The stamping + condition mirrors the cache-write keying (a hyperedge carries its own + ``source_file``); do not derive it from member nodes. + """ + root = Path(root) + + def _resolve(value: str) -> Path: + p = Path(value) + if not p.is_absolute(): + p = root / p + try: + return p.resolve() + except (OSError, RuntimeError): + return p + + sem_extracted: set[Path] = set() + for coll in ("nodes", "edges", "hyperedges"): + for item in sem_result.get(coll, []): + sf = item.get("source_file", "") + if sf: + sem_extracted.add(_resolve(sf)) + partial_resolved = {_resolve(p) for p in (partial_source_files or set())} + sem_types = {"document", "paper", "image"} + return { + ftype: [ + f for f in flist + if ftype not in sem_types + or (_resolve(f) in sem_extracted and _resolve(f) not in partial_resolved) + ] + for ftype, flist in files_by_type.items() + } + + +def _stale_graph_sources( + graph_path: Path, + scan_root: Path, + seen_files: set[str], + detection: dict | None = None, +) -> list[str]: + """Source files graph.json still references but the current scan no longer + contains (#1909). + + Incremental extract's prune set was historically derived from the manifest + alone (``manifest - corpus``), so a file that became EXCLUDED + (.graphifyignore/.gitignore/--exclude changed) without being listed in the + manifest kept its stale nodes in graph.json forever. Derive prune + candidates from the graph's own node ``source_file``s instead: anything + the graph references that the post-exclude detect corpus no longer + contains is stale, whether the file was deleted or newly excluded. + + Only IN-ROOT paths are candidates: out-of-root/absolute entries + (--include sources, symlinked external corpora) are never walked by + detect, so their absence from the corpus is not staleness evidence. + Relative entries are re-anchored against both the scan root and the + graph's own output root; only anchors that land inside the scan root + count. Since #1941 extracts always store source_file relative to the SCAN + root, so the scan-root anchor is the live one; the out-root anchor stays + for graphs written by <=0.9.16, which stored them relative to the OUT root + (e.g. ``../project/x.py``, #555/#1899). + ``seen_files`` must be the FULL detect output including unclassified + files, so nodes from walked-but-unsupported sources (e.g. introspected + Cargo.toml manifests) are not misread as stale. + + Paths are compared NFC-normalized on both sides: macOS reports NFD + filenames while graph ``source_file`` entries are typically NFC, and a + raw-string membership test misread every accented live file as stale + (#2210; same class as the manifest-layer #2221/#2224). + + Fail-closed liveness guard (#2210, mirrors watch.py's excluded-vs-deleted + distinction): a source missing from the scan corpus is only pruned when + the file is gone from disk, or when its exclusion is PROVABLE from the + same scan that produced ``seen_files`` — ``detection``'s ``ignored`` / + ``pruned_noise_dirs`` / ``skipped_sensitive`` output, or detect's + sensitivity predicate. An alive file that merely failed the membership + test (path-spelling drift the normalization didn't cover, walk errors, + …) is KEPT and reported, never mass-evicted. + """ + from graphify.paths import nfc + try: + data = json.loads(graph_path.read_text(encoding="utf-8")) + except Exception: + return [] + if not isinstance(data, dict): + return [] + try: + root_res = scan_root.resolve() + except (OSError, RuntimeError): + root_res = scan_root + # /graphify-out/graph.json — relative source_files may be anchored here. + out_base = graph_path.parent.parent + try: + out_base = out_base.resolve() + except (OSError, RuntimeError): + pass + + def _within_root(p: Path) -> bool: + try: + p.relative_to(root_res) + return True + except ValueError: + pass + try: + p.resolve().relative_to(root_res) + return True + except (ValueError, OSError, RuntimeError): + return False + + seen_nfc = {nfc(s) for s in seen_files} + seen_basenames = {nfc(os.path.basename(s)) for s in seen_files} + + def _in_seen(p: Path) -> bool: + if nfc(str(p)) in seen_nfc: + return True + try: + return nfc(str(p.resolve())) in seen_nfc + except (OSError, RuntimeError): + return False + + # Provable-exclusion evidence from the scan that produced seen_files: + # individually ignored files are exact entries; ignored/noise-pruned + # directories are recorded once with a trailing separator and cover + # their whole subtree. skipped_sensitive entries may carry a + # " [reason]" suffix. + excluded_exact: set[str] = set() + excluded_prefixes: list[str] = [] + if detection: + for entry in list(detection.get("ignored", [])) + list( + detection.get("pruned_noise_dirs", []) + ): + e = nfc(str(entry)) + if e.endswith(os.sep) or e.endswith("/"): + excluded_prefixes.append(e) + else: + excluded_exact.add(e) + for entry in detection.get("skipped_sensitive", []): + excluded_exact.add(nfc(str(entry).split(" [", 1)[0])) + + def _provably_excluded(c: Path) -> bool: + spellings = [nfc(str(c))] + try: + spellings.append(nfc(str(c.resolve()))) + except (OSError, RuntimeError): + pass + for s in spellings: + if s in excluded_exact: + return True + if any(s.startswith(pref) for pref in excluded_prefixes): + return True + try: + from graphify.detect import _is_sensitive as _det_sensitive + if _det_sensitive(c): + return True + except Exception: + pass + return False + + stale: list[str] = [] + kept_alive: list[str] = [] + checked: set[str] = set() + for n in data.get("nodes", []): + if not isinstance(n, dict): + continue + sf = n.get("source_file") + if not sf or not isinstance(sf, str) or sf in checked: + continue + checked.add(sf) + if "://" in sf: + continue # remote/virtual source (e.g. Google Workspace), not a scanned path + p = Path(sf) + if p.is_absolute(): + candidates = [p] + else: + rel = sf.replace("\\", "/") + bases = [root_res] + if out_base != root_res: + bases.append(out_base) + candidates = [ + Path(os.path.normpath(str(base / rel))) for base in bases + ] + in_root = [c for c in candidates if _within_root(c)] + if not in_root: + continue # out-of-root under every anchor: never prune + if any(_in_seen(c) for c in in_root): + continue # still part of the scan corpus + # Fail-closed liveness guard (#2210): absence from the corpus is + # only deletion evidence when the file is actually gone from disk. + alive = [] + for c in in_root: + try: + if c.exists(): + alive.append(c) + except OSError: + pass + if alive: + if all(_provably_excluded(c) for c in alive): + stale.append(sf) # alive but excluded under current rules (#1909) + else: + kept_alive.append(sf) + continue + # No anchored candidate exists, but a legacy bare-basename spelling + # can't be anchored reliably — a live corpus file with the same name + # means deletion is unproven; keep. + rel_sf = sf.replace("\\", "/") + if "/" not in rel_sf and nfc(rel_sf) in seen_basenames: + kept_alive.append(sf) + continue + stale.append(sf) + if kept_alive: + print( + f"[graphify] fail-closed: kept node(s) from {len(kept_alive)} " + "source file(s) that left the scan corpus but still exist on disk " + "(ignore rules or filters changed?). Run a full re-extraction to " + "purge them if the exclusion is intentional.", + file=sys.stderr, + ) + return stale + + +def _prune_graph_json_sources(graph_path: Path, stale_sources: list[str]) -> int: + """Drop nodes/edges/hyperedges owned by ``stale_sources`` from graph.json + in place. Returns the number of nodes removed. + + Used by the ``--no-cluster`` incremental early-exit: that path never runs + ``build_merge`` (it would raw-dump only the new chunks), so an + exclusion-only change must prune the existing raw graph directly or the + newly-excluded file's nodes survive forever (#1909). + ``stale_sources`` comes from :func:`_stale_graph_sources`, i.e. the + graph's own ``source_file`` spellings, so exact string matching is enough. + """ + try: + data = json.loads(graph_path.read_text(encoding="utf-8")) + except Exception: + return 0 + if not isinstance(data, dict): + return 0 + stale = set(stale_sources) + links_key = "links" if "links" in data else "edges" + nodes = [n for n in data.get("nodes", []) if isinstance(n, dict)] + kept_nodes = [n for n in nodes if n.get("source_file") not in stale] + removed_ids = { + n.get("id") for n in nodes if n.get("source_file") in stale + } + n_removed = len(nodes) - len(kept_nodes) + kept_edges = [ + e for e in data.get(links_key, []) + if isinstance(e, dict) + and e.get("source_file") not in stale + and e.get("source") not in removed_ids + and e.get("target") not in removed_ids + ] + kept_hyper = [ + h for h in data.get("hyperedges", []) + if isinstance(h, dict) and h.get("source_file") not in stale + ] + if n_removed == 0 and len(kept_edges) == len(data.get(links_key, [])) and ( + len(kept_hyper) == len(data.get("hyperedges", [])) + ): + return 0 + data["nodes"] = kept_nodes + data[links_key] = kept_edges + if "hyperedges" in data: + data["hyperedges"] = kept_hyper + from graphify.export import backup_if_protected as _backup + _backup(graph_path.parent) + from graphify.paths import write_json_atomic + write_json_atomic(graph_path, data, indent=2) + return n_removed + + +class _StageTimer: + """Print per-stage wall-clock timings to stderr when --timing is set (#1490). + + Monotonic (perf_counter), diagnostic-only: emits ``[graphify timing] : + N.Ns`` after each stage and a final total. Off by default, so normal output is + byte-identical and machine-read stdout is untouched. + """ + + def __init__(self, enabled: bool) -> None: + import time as _time + self._now = _time.perf_counter + self.enabled = enabled + self.start = self._now() + self._last = self.start + + def mark(self, stage: str) -> None: + now = self._now() + if self.enabled: + print(f"[graphify timing] {stage}: {now - self._last:.1f}s", file=sys.stderr) + self._last = now + + def total(self) -> None: + if self.enabled: + print(f"[graphify timing] total: {self._now() - self.start:.1f}s", file=sys.stderr) +def _enforce_graph_size_cap_or_exit(gp: Path) -> None: + """Reject oversized graph files before parsing (CLI exit-on-fail flavor). + + Delegates to ``graphify.security.check_graph_file_size_cap`` and turns the + raised ``ValueError`` into a CLI-style ``error: ...`` message + exit 1. + Use this from ``__main__.py`` subcommands that already use the ``print + + sys.exit(1)`` idiom. Library/MCP/loader callers (``serve._load_graph``, + ``build``, ``benchmark``, ``tree_html``, ``callflow_html``, ``prs``, + ``global_graph``, ``watch``, ``export``) call the security helper directly + and let the ``ValueError`` propagate. + """ + from graphify.security import check_graph_file_size_cap + try: + check_graph_file_size_cap(gp) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) +def _hook_strict_enabled(flag: bool) -> bool: + """Resolve strict mode: GRAPHIFY_HOOK_STRICT env overrides the baked-in flag + (truthy forces on without a reinstall, falsy is the kill switch); unset defers + to the flag the installed hook command carried.""" + v = os.environ.get("GRAPHIFY_HOOK_STRICT", "").strip().lower() + if v in ("1", "true", "yes", "on"): + return True + if v in ("0", "false", "no", "off"): + return False + return flag + + +def _touch_query_stamp(graph_path: "Path") -> None: + """Record that graphify oriented the agent recently, next to the queried graph. + The strict guard suppresses its block while this stamp is fresh. Fail-silent.""" + try: + from graphify.paths import write_text_atomic + stamp = Path(graph_path).parent / "cache" / "last_query_stamp" + stamp.parent.mkdir(parents=True, exist_ok=True) + write_text_atomic(stamp, str(time.time())) + except Exception: + pass + + +def _query_stamp_fresh() -> bool: + """True if a query/explain/path ran within GRAPHIFY_HOOK_STRICT_TTL (default + 1800s) — recent orientation, so strict mode does not block this read.""" + from graphify.paths import out_path + try: + ttl = float(os.environ.get("GRAPHIFY_HOOK_STRICT_TTL", "1800")) + return (time.time() - out_path("cache", "last_query_stamp").stat().st_mtime) < ttl + except Exception: + return False + + +def _mark_session_denied(session_id: str) -> bool: + """Atomically claim a one-time strict block for this session. Returns True only + on the FIRST call for a given session id (O_EXCL create wins once); every later + call — or any error — returns False, so a session is blocked at most once and an + agent can never be stranded. Best-effort GC of markers older than 24h.""" + from graphify.paths import out_path + sid = re.sub(r"[^A-Za-z0-9_-]", "_", str(session_id))[:64] + if not sid: + return False + try: + d = out_path("cache", "hook_sessions") + d.mkdir(parents=True, exist_ok=True) + fd = os.open(str(d / f"{sid}.denied"), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644) + os.close(fd) + try: + cutoff = time.time() - 86400 + for entry in os.scandir(d): + try: + if entry.stat().st_mtime < cutoff: + os.unlink(entry.path) + except OSError: + pass + except OSError: + pass + return True + except FileExistsError: + return False + except Exception: + return False + + +def _run_hook_guard(kind: str, strict: bool = False) -> None: + """Shell-agnostic PreToolUse guard (#522). + + Reads the tool-call JSON from stdin and, when a fresh in-project knowledge graph + exists, nudges the agent to use graphify instead of grepping/reading raw files. + Replaces the old inline bash hooks that failed to parse on Windows. + + Fails open everywhere: any error, or a non-matching tool call, prints nothing + and the caller exits 0, so a legitimate tool call is never blocked by a bug. + + In strict mode (opt-in, Claude Code Read only) the FIRST raw read of indexed, + in-project, fresh code per session is DENIED with a redirect to `graphify query` + (permissionDecision), then downgrades to the soft nudge — it fires at most once + per session and can never strand the agent. Search (Bash) and Glob stay + nudge-only: a compound shell command has no single parseable target and blocking + file listing would strand navigation. #1840: reads of out-of-project files are + ignored, and a graph that is stale for the target file softens to a non-mandatory + nudge instead of blocking or demanding. + """ + from graphify.paths import out_path, GRAPHIFY_OUT_NAME + # Gemini's BeforeTool hook takes no stdin and must ALWAYS return a decision so + # the tool is never blocked; the graph nudge is appended only when a graph + # exists. Handled before the stdin read below (which the search/read guards need). + if kind == "gemini": + payload = {"decision": "allow"} + try: + if out_path("graph.json").is_file(): + payload["additionalContext"] = _GEMINI_NUDGE_TEXT + except Exception: + pass + sys.stdout.write(json.dumps(payload, ensure_ascii=False, separators=(",", ":"))) + return + try: + d = json.loads(sys.stdin.buffer.read().decode("utf-8", "replace")) + except Exception: + return + if not isinstance(d, dict): + return + t = d.get("tool_input", d) + if not isinstance(t, dict): + return + try: + if kind == "search": + cmd_str = str(t.get("command", "") or "") + # Two input shapes reach this guard (matcher "Bash|Grep", #1986): + # the Bash tool carries `command`, while Claude Code's dedicated + # Grep tool carries `pattern` (plus optional path/glob) and no + # command — a Grep call IS a content search by definition, so it + # nudges whenever a graph exists. For Bash, keep matching the same + # set the old `case` matched: *grep*, *ripgrep*, and rg/find/fd/ + # ack/ag as a token (name followed by a space). Nudge-only, even in + # strict mode — see the docstring. + is_grep_tool = not cmd_str and bool(t.get("pattern")) + is_bash_search = any(tok in cmd_str for tok in ( + "grep", "ripgrep", "rg ", "find ", "fd ", "ack ", "ag ")) + if (is_grep_tool or is_bash_search) and out_path("graph.json").is_file(): + sys.stdout.write(_SEARCH_NUDGE) + elif kind == "read": + vals = [str(t.get("file_path") or ""), str(t.get("pattern") or ""), str(t.get("path") or "")] + j = " ".join(vals).lower().replace("\\", "/") + tails = [ + "." + seg.rsplit(".", 1)[-1] + for v in vals if v + for seg in [v.lower().replace("\\", "/").rsplit("/", 1)[-1]] + if "." in seg + ] + under_out = "graphify-out/" in j or (GRAPHIFY_OUT_NAME.lower() + "/") in j + if under_out or not any(tl in _HOOK_SOURCE_EXTS for tl in tails): + return + # #1840 (a): skip files outside the graph's project. cwd (or + # CLAUDE_PROJECT_DIR, which Claude Code sets) is the project root, since + # the guard only triggers when graph.json exists relative to cwd. A path + # candidate that resolves outside that root is out-of-project. + root = Path(os.environ.get("CLAUDE_PROJECT_DIR") or os.getcwd()) + try: + root = root.resolve() + except (OSError, RuntimeError): + pass + path_vals = [str(t.get("file_path") or ""), str(t.get("path") or "")] + explicit = [v for v in path_vals if v] + if explicit: + in_project = False + for v in explicit: + p = Path(v) + if not p.is_absolute(): + in_project = True # relative -> anchored at cwd == in project + break + try: + p.resolve().relative_to(root) + in_project = True + break + except (ValueError, OSError, RuntimeError): + continue + if not in_project: + return + # One stat for existence + mtime of the graph. + try: + gmtime = os.stat(str(out_path("graph.json"))).st_mtime + except OSError: + return + # #1840 (b): stale-for-target -> soften, never block. The target file + # changed after the last build, or watch flagged the tree. + stale = False + fp = str(t.get("file_path") or "") + if fp: + try: + stale = os.stat(fp).st_mtime > gmtime + except OSError: + stale = False + try: + if out_path("needs_update").exists(): + stale = True + except Exception: + pass + if stale: + sys.stdout.write(_READ_NUDGE_STALE) + return + # Strict block: Read tool only, first time per session, not recently + # oriented, and the file is demonstrably indexed. + tool_name = d.get("tool_name") + if _hook_strict_enabled(strict) and tool_name in (None, "Read") \ + and not _query_stamp_fresh() \ + and _target_is_indexed(fp, root) \ + and _mark_session_denied(str(d.get("session_id") or "")): + sys.stdout.write(_READ_DENY) + return + sys.stdout.write(_READ_NUDGE) + except Exception: + pass + + +def _target_is_indexed(file_path: str, root: "Path") -> bool: + """Guard the strict deny: only block a read of a file the graph actually indexes. + Reads manifest.json (cheap, capped); on any doubt (missing/corrupt/oversized + manifest, unresolvable path) returns True so the once-per-session deny still + applies — that block is self-limiting, so erring toward it is safe.""" + from graphify.paths import out_path + if not file_path: + return True + try: + mp = out_path("manifest.json") + st = mp.stat() + if st.st_size > 2_000_000: + return True + manifest = json.loads(mp.read_text(encoding="utf-8")) + if not isinstance(manifest, dict) or not manifest: + return True + p = Path(file_path) + rels = set() + try: + rels.add(p.resolve().relative_to(root).as_posix()) + except (ValueError, OSError, RuntimeError): + pass + rels.add(p.name) + keys = {str(k).replace("\\", "/") for k in manifest} + abskey = str(p).replace("\\", "/") + return abskey in keys or any(r and (r in keys or any(k.endswith("/" + r) or k == r for k in keys)) for r in rels) + except Exception: + return True +def _clone_repo( + url: str, branch: str | None = None, out_dir: Path | None = None +) -> Path: + """Clone a GitHub repo to a local cache dir and return the path. + + Clones into ~/.graphify/repos// by default so repeated + runs on the same URL reuse the existing clone (git pull instead of clone). + """ + import subprocess as _sp + import re as _re + + # Normalise URL — strip trailing .git if present + url = url.rstrip("/") + if not url.endswith(".git"): + git_url = url + ".git" + else: + git_url = url + url = url[:-4] + + # Extract owner/repo from URL + m = _re.search(r"github\.com[:/]([^/]+)/([^/]+?)(?:\.git)?$", url) + if not m: + print(f"error: not a recognised GitHub URL: {url}", file=sys.stderr) + sys.exit(1) + owner, repo = m.group(1), m.group(2) + + if out_dir: + dest = out_dir + else: + dest = Path.home() / ".graphify" / "repos" / owner / repo + + if branch and branch.startswith("-"): + print(f"error: invalid branch name: {branch!r}", file=sys.stderr) + sys.exit(1) + + if dest.exists(): + print(f"Repo already cloned at {dest} - pulling latest...", flush=True) + cmd = ["git", "-C", str(dest), "pull"] + if branch: + cmd += ["origin", "--", branch] + result = _sp.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"warning: git pull failed:\n{result.stderr}", file=sys.stderr) + else: + dest.parent.mkdir(parents=True, exist_ok=True) + print(f"Cloning {url} -> {dest} ...", flush=True) + cmd = ["git", "clone", "--depth", "1"] + if branch: + cmd += ["--branch", branch] + cmd += ["--", git_url, str(dest)] + result = _sp.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print(f"error: git clone failed:\n{result.stderr}", file=sys.stderr) + sys.exit(1) + + print(f"Ready at: {dest}", flush=True) + return dest + + +def _reenter_main() -> None: + from graphify.__main__ import main + main() + + +def dispatch_command(cmd: str) -> None: + if cmd == "provider": + from graphify.llm import _custom_providers_path, BACKENDS + import json as _json + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + global_path = _custom_providers_path(global_=True) + + if subcmd == "list": + global_path.parent.mkdir(parents=True, exist_ok=True) + existing: dict = {} + if global_path.is_file(): + try: + existing = _json.loads(global_path.read_text(encoding="utf-8")) + except Exception: + pass + if not existing: + print("No custom providers registered.") + else: + for name in existing: + print(f" {name} ({existing[name].get('base_url', '')})") + + elif subcmd == "show": + name = sys.argv[3] if len(sys.argv) > 3 else "" + if not name: + print("Usage: graphify provider show ", file=sys.stderr) + sys.exit(1) + existing = {} + if global_path.is_file(): + try: + existing = _json.loads(global_path.read_text(encoding="utf-8")) + except Exception: + pass + if name not in existing: + print(f"Provider '{name}' not found.", file=sys.stderr) + sys.exit(1) + print(_json.dumps({name: existing[name]}, indent=2)) + + elif subcmd == "add": + args = sys.argv[3:] + name = args[0] if args and not args[0].startswith("-") else "" + if not name: + print("Usage: graphify provider add --base-url URL --default-model MODEL --env-key KEY", file=sys.stderr) + sys.exit(1) + if name in BACKENDS: + print(f"Error: '{name}' is a built-in provider and cannot be overridden.", file=sys.stderr) + sys.exit(1) + base_url = "" + default_model = "" + env_key = "" + pricing_input = 0.0 + pricing_output = 0.0 + i = 1 + while i < len(args): + a = args[i] + if a == "--base-url" and i + 1 < len(args): + base_url = args[i + 1]; i += 2 + elif a.startswith("--base-url="): + base_url = a.split("=", 1)[1]; i += 1 + elif a == "--default-model" and i + 1 < len(args): + default_model = args[i + 1]; i += 2 + elif a.startswith("--default-model="): + default_model = a.split("=", 1)[1]; i += 1 + elif a == "--env-key" and i + 1 < len(args): + env_key = args[i + 1]; i += 2 + elif a.startswith("--env-key="): + env_key = a.split("=", 1)[1]; i += 1 + elif a == "--pricing-input" and i + 1 < len(args): + pricing_input = float(args[i + 1]); i += 2 + elif a == "--pricing-output" and i + 1 < len(args): + pricing_output = float(args[i + 1]); i += 2 + else: + i += 1 + if not base_url or not default_model or not env_key: + print("Error: --base-url, --default-model, and --env-key are required.", file=sys.stderr) + sys.exit(1) + from graphify.llm import provider_base_url_ok + if not provider_base_url_ok(base_url, name): + print(f"Error: refusing to add provider with unsafe base_url {base_url!r}.", file=sys.stderr) + sys.exit(1) + global_path.parent.mkdir(parents=True, exist_ok=True) + existing = {} + if global_path.is_file(): + try: + existing = _json.loads(global_path.read_text(encoding="utf-8")) + except Exception: + pass + existing[name] = { + "base_url": base_url, + "default_model": default_model, + "env_key": env_key, + "pricing": {"input": pricing_input, "output": pricing_output}, + "temperature": 0, + } + global_path.write_text(_json.dumps(existing, indent=2) + "\n", encoding="utf-8") + print(f"Provider '{name}' added. Use with: graphify extract . --backend {name}") + + elif subcmd == "remove": + name = sys.argv[3] if len(sys.argv) > 3 else "" + if not name: + print("Usage: graphify provider remove ", file=sys.stderr) + sys.exit(1) + existing = {} + if global_path.is_file(): + try: + existing = _json.loads(global_path.read_text(encoding="utf-8")) + except Exception: + pass + if name not in existing: + print(f"Provider '{name}' not found.", file=sys.stderr) + sys.exit(1) + del existing[name] + global_path.write_text(_json.dumps(existing, indent=2) + "\n", encoding="utf-8") + print(f"Provider '{name}' removed.") + + else: + print("Usage: graphify provider [add|list|show|remove]", file=sys.stderr) + if subcmd: + sys.exit(1) + elif cmd == "prs": + from graphify.prs import cmd_prs + cmd_prs(sys.argv[2:]) + elif cmd == "hook": + from graphify.hooks import ( + install as hook_install, + uninstall as hook_uninstall, + status as hook_status, + ) + + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + if subcmd == "install": + print(hook_install(Path("."))) + elif subcmd == "uninstall": + print(hook_uninstall(Path("."))) + elif subcmd == "status": + print(hook_status(Path("."))) + else: + print("Usage: graphify hook [install|uninstall|status]", file=sys.stderr) + sys.exit(1) + elif cmd == "query": + if len(sys.argv) < 3: + print("Usage: graphify query \"\" [--dfs] [--context C] [--budget N] [--graph path]", file=sys.stderr) + sys.exit(1) + from graphify.serve import _query_graph_text + from graphify.security import sanitize_label + from networkx.readwrite import json_graph + from graphify import querylog + + question = sys.argv[2] + use_dfs = "--dfs" in sys.argv + budget = 2000 + graph_path = _default_graph_path() + context_filters: list[str] = [] + args = sys.argv[3:] + i = 0 + while i < len(args): + if args[i] == "--budget" and i + 1 < len(args): + try: + budget = int(args[i + 1]) + except ValueError: + print(f"error: --budget must be an integer", file=sys.stderr) + sys.exit(1) + i += 2 + elif args[i].startswith("--budget="): + try: + budget = int(args[i].split("=", 1)[1]) + except ValueError: + print(f"error: --budget must be an integer", file=sys.stderr) + sys.exit(1) + i += 1 + elif args[i] == "--context" and i + 1 < len(args): + context_filters.append(args[i + 1]) + i += 2 + elif args[i].startswith("--context="): + context_filters.append(args[i].split("=", 1)[1]) + i += 1 + elif args[i] == "--graph" and i + 1 < len(args): + graph_path = args[i + 1] + i += 2 + else: + i += 1 + gp = Path(graph_path).resolve() + if not gp.exists(): + print(f"error: graph file not found: {gp}", file=sys.stderr) + sys.exit(1) + if not gp.suffix == ".json": + print(f"error: graph file must be a .json file", file=sys.stderr) + sys.exit(1) + _enforce_graph_size_cap_or_exit(gp) + try: + import json as _json + import networkx as _nx + + _raw = _json.loads(gp.read_text(encoding="utf-8")) + if "links" not in _raw and "edges" in _raw: + _raw = dict(_raw, links=_raw["edges"]) + # `query` deliberately keeps the graph undirected (unlike `path` / + # `explain`, which force directed=True): BFS/DFS here must explore + # both callers and callees of the seed node to build useful + # context, and forcing a DiGraph would make G.neighbors() return + # successors only, silently dropping every caller-side result for + # a seed with no outgoing edges. Direction is instead preserved + # per-edge below (mirrors graphify/build.py's _src/_tgt pattern) + # so the *rendering* stays correct without narrowing traversal. + _raw = dict( + _raw, + links=[ + {**link, "_src": link.get("source"), "_tgt": link.get("target")} + for link in _raw.get("links", []) + ], + ) + try: + G = json_graph.node_link_graph(_raw, edges="links") + except TypeError: + G = json_graph.node_link_graph(_raw) + try: + from graphify.build import graph_has_legacy_ids as _legacy + if _legacy(_raw.get("nodes", [])): + print( + "[graphify] note: this graph uses the pre-#1504 node-ID scheme; " + "rebuild with `graphify extract --force` to get path-qualified IDs " + "(fixes same-name-file collisions).", + file=sys.stderr, + ) + except Exception: + pass + except Exception as exc: + print(f"error: could not load graph: {exc}", file=sys.stderr) + sys.exit(1) + import time as _time + _t0 = _time.perf_counter() + _mode = "dfs" if use_dfs else "bfs" + _result = _query_graph_text( + G, + question, + mode=_mode, + depth=2, + token_budget=budget, + context_filters=context_filters, + ) + querylog.log_query( + kind="query", + question=question, + corpus=str(gp), + result=_result, + mode=_mode, + depth=2, + token_budget=budget, + duration_ms=(_time.perf_counter() - _t0) * 1000, + ) + _touch_query_stamp(gp) + print(_result) + elif cmd == "affected": + if len(sys.argv) < 3: + print("Usage: graphify affected \"\" [--relation R] [--depth N] [--graph path]", file=sys.stderr) + sys.exit(1) + from graphify.affected import DEFAULT_AFFECTED_RELATIONS, format_affected, load_graph + query = sys.argv[2] + graph_path = _default_graph_path() + depth = 2 + relations: list[str] = [] + args = sys.argv[3:] + i = 0 + while i < len(args): + if args[i] == "--graph" and i + 1 < len(args): + graph_path = args[i + 1] + i += 2 + elif args[i].startswith("--graph="): + graph_path = args[i].split("=", 1)[1] + i += 1 + elif args[i] == "--depth" and i + 1 < len(args): + try: + depth = int(args[i + 1]) + except ValueError: + print("error: --depth must be an integer", file=sys.stderr) + sys.exit(1) + i += 2 + elif args[i].startswith("--depth="): + try: + depth = int(args[i].split("=", 1)[1]) + except ValueError: + print("error: --depth must be an integer", file=sys.stderr) + sys.exit(1) + i += 1 + elif args[i] == "--relation" and i + 1 < len(args): + relations.append(args[i + 1]) + i += 2 + elif args[i].startswith("--relation="): + relations.append(args[i].split("=", 1)[1]) + i += 1 + else: + i += 1 + gp = Path(graph_path).resolve() + if not gp.exists(): + print(f"error: graph file not found: {gp}", file=sys.stderr) + sys.exit(1) + if not gp.suffix == ".json": + print("error: graph file must be a .json file", file=sys.stderr) + sys.exit(1) + try: + graph = load_graph(gp) + except Exception as exc: + print(f"error: could not load graph: {exc}", file=sys.stderr) + sys.exit(1) + print( + format_affected( + graph, + query, + relations=relations or DEFAULT_AFFECTED_RELATIONS, + depth=depth, + ) + ) + elif cmd in ("god-nodes", "god_nodes"): + # god_nodes has long been an analyzer (analyze.py), an MCP tool, and a + # README-advertised capability, but never a CLI subcommand — `graphify + # god_nodes` fell through to "unknown command" (#2004). Wire it as a + # read-only graph query, mirroring `affected`. + from graphify.affected import load_graph + from graphify.analyze import god_nodes as _god_nodes + from graphify.security import sanitize_label as _sanitize_label + graph_path = _default_graph_path() + top_n = 10 + as_json = "--json" in sys.argv + args = sys.argv[2:] + i = 0 + while i < len(args): + if args[i] == "--graph" and i + 1 < len(args): + graph_path = args[i + 1] + i += 2 + elif args[i].startswith("--graph="): + graph_path = args[i].split("=", 1)[1] + i += 1 + elif args[i] == "--top" and i + 1 < len(args): + try: + top_n = int(args[i + 1]) + except ValueError: + print("error: --top must be an integer", file=sys.stderr) + sys.exit(1) + i += 2 + elif args[i].startswith("--top="): + try: + top_n = int(args[i].split("=", 1)[1]) + except ValueError: + print("error: --top must be an integer", file=sys.stderr) + sys.exit(1) + i += 1 + else: + i += 1 + gp = Path(graph_path).resolve() + if not gp.exists(): + print(f"error: graph file not found: {gp}", file=sys.stderr) + sys.exit(1) + if not gp.suffix == ".json": + print("error: graph file must be a .json file", file=sys.stderr) + sys.exit(1) + try: + G = load_graph(gp) + except Exception as exc: + print(f"error: could not load graph: {exc}", file=sys.stderr) + sys.exit(1) + gods = _god_nodes(G, top_n=top_n) + if as_json: + print(json.dumps(gods, indent=2)) + else: + print("God nodes (most connected):") + for rank, n in enumerate(gods, 1): + print(f" {rank}. {_sanitize_label(str(n['label']))} - {n['degree']} edges") + elif cmd == "save-result": + # graphify save-result --question Q --answer A [--type T] [--nodes N1 N2 ...] + # [--outcome useful|dead_end|corrected] [--correction TEXT] + import argparse as _ap + + p = _ap.ArgumentParser(prog="graphify save-result") + p.add_argument("--question", required=True) + p.add_argument("--answer", default=None) + p.add_argument("--answer-file", dest="answer_file", default=None) + p.add_argument("--type", dest="query_type", default="query") + p.add_argument("--nodes", nargs="*", default=[]) + p.add_argument("--outcome", choices=("useful", "dead_end", "corrected"), default=None) + p.add_argument("--correction", default=None) + p.add_argument("--memory-dir", default=str(Path(_GRAPHIFY_OUT) / "memory")) + opts = p.parse_args(sys.argv[2:]) + if opts.answer_file: + opts.answer = Path(opts.answer_file).read_text(encoding="utf-8").strip() + elif not opts.answer: + p.error("--answer or --answer-file is required") + from graphify.ingest import save_query_result as _sqr + + out = _sqr( + question=opts.question, + answer=opts.answer, + memory_dir=Path(opts.memory_dir), + query_type=opts.query_type, + source_nodes=opts.nodes or None, + outcome=opts.outcome, + correction=opts.correction, + ) + print(f"Saved to {out}") + elif cmd == "reflect": + import argparse as _ap + + p = _ap.ArgumentParser(prog="graphify reflect") + p.add_argument("--memory-dir", default=str(Path(_GRAPHIFY_OUT) / "memory")) + p.add_argument( + "--out", + default=str(Path(_GRAPHIFY_OUT) / "reflections" / "LESSONS.md"), + ) + p.add_argument("--graph", default=None) + p.add_argument("--analysis", default=None) + p.add_argument("--labels", default=None) + p.add_argument("--half-life-days", type=float, default=30.0, + help="signal weight halves every N days (default 30)") + p.add_argument("--min-corroboration", type=int, default=2, + help="distinct useful results to promote a node to preferred (default 2)") + p.add_argument("--if-stale", action="store_true", + help="skip when LESSONS.md is already newer than every input " + "(e.g. the git hook just refreshed it)") + opts = p.parse_args(sys.argv[2:]) + from graphify.reflect import reflect as _reflect, lessons_fresh as _lessons_fresh + + graph_arg = opts.graph + if graph_arg is None: + default_graph = Path(_GRAPHIFY_OUT) / "graph.json" + if default_graph.exists(): + graph_arg = str(default_graph) + + _gp = Path(graph_arg) if graph_arg else None + _analysis_path = None + _labels_path = None + if _gp is not None: + _analysis_path = Path(opts.analysis) if opts.analysis else ( + _gp.parent / ".graphify_analysis.json") + _labels_path = Path(opts.labels) if opts.labels else ( + _gp.parent / ".graphify_labels.json") + + if opts.if_stale and _lessons_fresh( + Path(opts.out), Path(opts.memory_dir), _gp, _analysis_path, _labels_path + ): + print(f"Lessons already up to date -> {opts.out} (skipped; omit --if-stale to force)") + else: + out_path, agg = _reflect( + memory_dir=Path(opts.memory_dir), + out_path=Path(opts.out), + graph_path=_gp, + analysis_path=_analysis_path, + labels_path=_labels_path, + half_life_days=opts.half_life_days, + min_corroboration=opts.min_corroboration, + ) + c = agg["counts"] + print( + f"Reflected {agg['total']} memories " + f"({c['useful']} useful, {c['dead_end']} dead ends, " + f"{c['corrected']} corrected) -> {out_path}" + ) + elif cmd == "path": + if len(sys.argv) < 4: + print( + 'Usage: graphify path "" "" [--graph path]', + file=sys.stderr, + ) + sys.exit(1) + from graphify.serve import _pick_scored_endpoint, _score_nodes + from networkx.readwrite import json_graph + import networkx as _nx + + source_label = sys.argv[2] + target_label = sys.argv[3] + graph_path = _default_graph_path() + args = sys.argv[4:] + for i, a in enumerate(args): + if a == "--graph" and i + 1 < len(args): + graph_path = args[i + 1] + gp = Path(graph_path).resolve() + if not gp.exists(): + print(f"error: graph file not found: {gp}", file=sys.stderr) + sys.exit(1) + _enforce_graph_size_cap_or_exit(gp) + _raw = json.loads(gp.read_text(encoding="utf-8")) + if "links" not in _raw and "edges" in _raw: + _raw = dict(_raw, links=_raw["edges"]) + # Force directed so the renderer can recover stored caller→callee + # direction, and multigraph so exact-pair parallel links (e.g. a + # `references` and a `calls` edge between the same two nodes) survive load + # instead of being silently collapsed last-writer-wins — otherwise the + # printed relation could be one the traversed pair doesn't actually + # carry (#2074). Local to this read; serve's shared graph is untouched. + _raw = {**_raw, "directed": True, "multigraph": True} + try: + G = json_graph.node_link_graph(_raw, edges="links") + except TypeError: + G = json_graph.node_link_graph(_raw) + src_scored = _score_nodes(G, [t.lower() for t in source_label.split()]) + tgt_scored = _score_nodes(G, [t.lower() for t in target_label.split()]) + if not src_scored: + print(f"No node matching '{source_label}' found.", file=sys.stderr) + sys.exit(1) + if not tgt_scored: + print(f"No node matching '{target_label}' found.", file=sys.stderr) + sys.exit(1) + src_nid = _pick_scored_endpoint(G, src_scored, source_label) + tgt_nid = _pick_scored_endpoint(G, tgt_scored, target_label) + # Ambiguity guard: when both queries resolve to the same node, the + # shortest path is trivially zero hops, which is almost never what the + # caller wanted (see bug #828). + if src_nid == tgt_nid: + print( + f"'{source_label}' and '{target_label}' both resolved to the same " + f"node '{src_nid}'. Use a more specific label or the exact node ID.", + file=sys.stderr, + ) + sys.exit(1) + for _name, _scored, _nid in ( + ("source", src_scored, src_nid), + ("target", tgt_scored, tgt_nid), + ): + # A close runner-up only made the resolution ambiguous when the raw + # score head is what got picked; a full-token override was chosen on + # token coverage, not score, so the head's margin is irrelevant. + if len(_scored) >= 2 and _nid == _scored[0][1]: + _top, _runner = _scored[0][0], _scored[1][0] + if _top > 0 and (_top - _runner) / _top < 0.10: + print( + f"warning: {_name} match was ambiguous " + f"(top score {_top:g}, runner-up {_runner:g})", + file=sys.stderr, + ) + # Deterministic shortest path (#2074): to_undirected(as_view=True) + # iterates neighbors via a hash-seeded set union, so among equal-length + # paths BFS returned an arbitrary route that varied per process. Build a + # sorted, materialized undirected graph so neighbor order — and thus the + # chosen path — is canonical for a given graph.json. + _und = _nx.Graph() + _und.add_nodes_from(sorted(G.nodes)) + _und.add_edges_from(sorted((min(u, v), max(u, v)) for u, v in G.edges())) + try: + path_nodes = _nx.shortest_path(_und, src_nid, tgt_nid) + except (_nx.NetworkXNoPath, _nx.NodeNotFound): + print(f"No path found between '{source_label}' and '{target_label}'.") + sys.exit(0) + hops = len(path_nodes) - 1 + segments = [] + from graphify.build import edge_datas + for i in range(len(path_nodes) - 1): + u, v = path_nodes[i], path_nodes[i + 1] + # Report the ACTUAL stored relation(s) of the traversed pair and + # direction — never a fabricated `calls` (#2074). A pair may carry + # several parallel relations; show all, and fall back to an honest + # "related" when the stored edge has no relation. + if G.has_edge(u, v): + datas = edge_datas(G, u, v) + forward = True + else: + datas = edge_datas(G, v, u) + forward = False + rels = sorted({d.get("relation") for d in datas if d.get("relation")}) + rel = "/".join(rels) if rels else "related" + confs = sorted({d.get("confidence") for d in datas if d.get("confidence")}) + conf_str = f" [{'/'.join(confs)}]" if confs else "" + if i == 0: + segments.append(G.nodes[u].get("label", u)) + if forward: + segments.append(f"--{rel}{conf_str}--> {G.nodes[v].get('label', v)}") + else: + segments.append(f"<--{rel}{conf_str}-- {G.nodes[v].get('label', v)}") + print(f"Shortest path ({hops} hops):\n " + " ".join(segments)) + from graphify import querylog + querylog.log_query( + kind="path", + question=f"{sys.argv[2]} -> {sys.argv[3]}", + corpus=str(gp), + nodes_returned=hops, + ) + _touch_query_stamp(gp) + + elif cmd == "explain": + if len(sys.argv) < 3: + print('Usage: graphify explain "" [--graph path]', file=sys.stderr) + sys.exit(1) + from graphify.serve import _find_node + from networkx.readwrite import json_graph + + label = sys.argv[2] + graph_path = _default_graph_path() + args = sys.argv[3:] + for i, a in enumerate(args): + if a == "--graph" and i + 1 < len(args): + graph_path = args[i + 1] + gp = Path(graph_path).resolve() + if not gp.exists(): + print(f"error: graph file not found: {gp}", file=sys.stderr) + sys.exit(1) + _enforce_graph_size_cap_or_exit(gp) + _raw = json.loads(gp.read_text(encoding="utf-8")) + if "links" not in _raw and "edges" in _raw: + _raw = dict(_raw, links=_raw["edges"]) + # Force directed so the renderer can recover stored caller→callee direction. + _raw = {**_raw, "directed": True} + try: + G = json_graph.node_link_graph(_raw, edges="links") + except TypeError: + G = json_graph.node_link_graph(_raw) + matches = _find_node(G, label) + if not matches: + print(f"No node matching '{label}' found.") + sys.exit(0) + nid = matches[0] + d = G.nodes[nid] + print(f"Node: {d.get('label', nid)}") + print(f" ID: {nid}") + print( + f" Source: {d.get('source_file', '')} {d.get('source_location', '')}".rstrip() + ) + print(f" Type: {d.get('file_type', '')}") + print(f" Community: {d.get('community_name') or d.get('community', '')}") + # Work-memory overlay: a derived experiential hint from `graphify reflect`, + # merged in display-only from the .graphify_learning.json sidecar next to + # graph.json. No line when the node has no overlay entry. + try: + from graphify.reflect import load_learning_overlay as _llo + from graphify.security import sanitize_label as _sl + _overlay = _llo(gp) + _entry = _overlay.get(str(nid)) + if _entry: + _status = _sl(str(_entry.get("status", ""))) + if _status == "contested": + _line = (f" Lesson: contested (useful {_entry.get('uses', 0)} / " + f"dead-end {_entry.get('neg', 0)})") + elif _status == "preferred": + _line = (f" Lesson: preferred source (start here) — " + f"{_entry.get('uses', 0)} useful, score={_entry.get('score', 0)}") + else: + _line = (f" Lesson: {_status or 'tentative'} — " + f"{_entry.get('uses', 0)} useful, score={_entry.get('score', 0)}") + if _entry.get("stale"): + _line += " [code changed since — re-verify]" + print(_line) + except Exception: + pass + print(f" Degree: {G.degree(nid)}") + from graphify.build import edge_data + connections: list[tuple[str, str, dict]] = [] # (direction, neighbor_id, edge_data) + for nb in G.successors(nid): + connections.append(("out", nb, edge_data(G, nid, nb))) + for nb in G.predecessors(nid): + connections.append(("in", nb, edge_data(G, nb, nid))) + if connections: + print(f"\nConnections ({len(connections)}):") + connections.sort(key=lambda c: G.degree(c[1]), reverse=True) + for direction, nb, edata in connections[:20]: + rel = edata.get("relation", "") + conf = edata.get("confidence", "") + arrow = "-->" if direction == "out" else "<--" + # Append the edge's location — the actual call/import/reference + # SITE (in the caller's file for an incoming call), not a def + # line (#BUG1). Labeled by [rel] so the meaning is unambiguous. + loc = edata.get("source_location") or "" + sfile = edata.get("source_file") or "" + at = f" {sfile}:{loc}" if loc else "" + print(f" {arrow} {G.nodes[nb].get('label', nb)} [{rel}] [{conf}]{at}") + if len(connections) > 20: + remainder = connections[20:] + print(f" ... and {len(remainder)} more") + # #2009: a bare count silently hides the answer on high-degree + # nodes ("who calls this, what's the impact?"). Group the cut + # connections by direction + file so their shape is visible + # without falling back to a repo-wide grep. + by_file: dict[tuple[str, str], int] = {} + for direction, _nb, edata in remainder: + sfile = edata.get("source_file") or "(unknown file)" + key = (direction, sfile) + by_file[key] = by_file.get(key, 0) + 1 + # Count desc, then (direction, file) so equal-count groups have a + # byte-stable order (not the degree-derived insertion order). + grouped = sorted(by_file.items(), key=lambda kv: (-kv[1], kv[0])) + print(" Grouped by file:") + for (direction, sfile), count in grouped[:20]: + arrow = "-->" if direction == "out" else "<--" + noun = "connection" if count == 1 else "connections" + print(f" {arrow} {sfile}: {count} {noun}") + if len(grouped) > 20: + print(f" ... and {len(grouped) - 20} more files") + from graphify import querylog + querylog.log_query( + kind="explain", + question=sys.argv[2], + corpus=str(gp), + nodes_returned=len(connections), + ) + _touch_query_stamp(gp) + + elif cmd == "diagnose": + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + if subcmd != "multigraph": + print( + "Usage: graphify diagnose multigraph " + "[--graph path] [--json] [--max-examples N] " + "[--directed] [--undirected] [--extract-path path]", + file=sys.stderr, + ) + sys.exit(1) + + graph_path = Path(_default_graph_path()) + max_examples = 5 + directed: bool | None = None + direction_flag: str | None = None + json_output = False + extract_path: Path | None = None + + i = 3 + while i < len(sys.argv): + arg = sys.argv[i] + if arg == "--graph": + i += 1 + if i >= len(sys.argv): + print("error: --graph requires a path", file=sys.stderr) + sys.exit(1) + graph_path = Path(sys.argv[i]) + elif arg == "--json": + json_output = True + elif arg == "--max-examples": + i += 1 + if i >= len(sys.argv): + print("error: --max-examples requires an integer", file=sys.stderr) + sys.exit(1) + try: + max_examples = int(sys.argv[i]) + except ValueError: + print("error: --max-examples requires an integer", file=sys.stderr) + sys.exit(1) + if max_examples < 0: + print("error: --max-examples must be >= 0", file=sys.stderr) + sys.exit(1) + elif arg == "--directed": + if direction_flag == "undirected": + print( + "error: --directed and --undirected are mutually exclusive", + file=sys.stderr, + ) + sys.exit(1) + direction_flag = "directed" + directed = True + elif arg == "--undirected": + if direction_flag == "directed": + print( + "error: --directed and --undirected are mutually exclusive", + file=sys.stderr, + ) + sys.exit(1) + direction_flag = "undirected" + directed = False + elif arg == "--extract-path": + i += 1 + if i >= len(sys.argv): + print("error: --extract-path requires a path", file=sys.stderr) + sys.exit(1) + extract_path = Path(sys.argv[i]) + else: + print(f"error: unknown diagnose option {arg}", file=sys.stderr) + sys.exit(1) + i += 1 + + from graphify.diagnostics import ( + diagnose_file, + format_diagnostic_json, + format_diagnostic_report, + ) + + try: + summary = diagnose_file( + graph_path, + directed=directed, + root=Path(".").resolve(), + max_examples=max_examples, + extract_path=extract_path, + ) + except Exception as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + + if json_output: + print(json.dumps(format_diagnostic_json(summary), indent=2)) + else: + print(format_diagnostic_report(summary)) + + elif cmd == "add": + if len(sys.argv) < 3: + print( + "Usage: graphify add [--author Name] [--contributor Name] [--dir ./raw]", + file=sys.stderr, + ) + sys.exit(1) + from graphify.ingest import ingest as _ingest + + url = sys.argv[2] + author: str | None = None + contributor: str | None = None + target_dir = Path("raw") + args = sys.argv[3:] + i = 0 + while i < len(args): + if args[i] == "--author" and i + 1 < len(args): + author = args[i + 1] + i += 2 + elif args[i] == "--contributor" and i + 1 < len(args): + contributor = args[i + 1] + i += 2 + elif args[i] == "--dir" and i + 1 < len(args): + target_dir = Path(args[i + 1]) + i += 2 + else: + i += 1 + try: + saved = _ingest(url, target_dir, author=author, contributor=contributor) + print(f"Saved to {saved}") + print("Run /graphify --update in your AI assistant to update the graph.") + except Exception as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + + elif cmd == "watch": + watch_path = Path(sys.argv[2]) if len(sys.argv) > 2 else Path(".") + if not watch_path.exists(): + print(f"error: path not found: {watch_path}", file=sys.stderr) + sys.exit(1) + from graphify.watch import watch as _watch + + try: + _watch(watch_path) + except ImportError as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + + elif cmd in ("cluster-only", "label"): + # `label` is `cluster-only` that always (re)generates community names with + # the configured backend, even when a .graphify_labels.json already exists. + force_relabel = cmd == "label" + # Mirror the tree/export arg-parsing pattern: walk argv so flags and + # the optional positional path can appear in any order (#724). + no_viz = "--no-viz" in sys.argv + no_label = "--no-label" in sys.argv + missing_only = "--missing-only" in sys.argv + co_timing = "--timing" in sys.argv + _backend_arg = next((a for a in sys.argv if a.startswith("--backend=")), None) + label_backend = _backend_arg.split("=", 1)[1] if _backend_arg else None + _model_arg = next((a for a in sys.argv if a.startswith("--model=")), None) + label_model = _model_arg.split("=", 1)[1] if _model_arg else None + _min_cs_arg = next((a for a in sys.argv if a.startswith("--min-community-size=")), None) + min_community_size = int(_min_cs_arg.split("=")[1]) if _min_cs_arg else 3 + args = sys.argv[2:] + watch_path: Path | None = None + graph_override: Path | None = None + co_resolution: float = 1.0 + co_exclude_hubs: float | None = None + label_max_concurrency: int = 4 + label_batch_size: int = 100 + i_arg = 0 + while i_arg < len(args): + a = args[i_arg] + if a == "--graph" and i_arg + 1 < len(args): + graph_override = Path(args[i_arg + 1]); i_arg += 2 + elif a == "--backend" and i_arg + 1 < len(args): + label_backend = args[i_arg + 1]; i_arg += 2 + elif a.startswith("--backend="): + label_backend = a.split("=", 1)[1]; i_arg += 1 + elif a == "--model" and i_arg + 1 < len(args): + label_model = args[i_arg + 1]; i_arg += 2 + elif a.startswith("--model="): + label_model = a.split("=", 1)[1]; i_arg += 1 + elif a == "--resolution" and i_arg + 1 < len(args): + co_resolution = float(args[i_arg + 1]); i_arg += 2 + elif a.startswith("--resolution="): + co_resolution = float(a.split("=", 1)[1]); i_arg += 1 + elif a == "--exclude-hubs" and i_arg + 1 < len(args): + co_exclude_hubs = float(args[i_arg + 1]); i_arg += 2 + elif a.startswith("--exclude-hubs="): + co_exclude_hubs = float(a.split("=", 1)[1]); i_arg += 1 + elif a == "--max-concurrency" and i_arg + 1 < len(args): + label_max_concurrency = int(args[i_arg + 1]); i_arg += 2 + elif a.startswith("--max-concurrency="): + label_max_concurrency = int(a.split("=", 1)[1]); i_arg += 1 + elif a == "--batch-size" and i_arg + 1 < len(args): + label_batch_size = int(args[i_arg + 1]); i_arg += 2 + elif a.startswith("--batch-size="): + label_batch_size = int(a.split("=", 1)[1]); i_arg += 1 + elif a in ("--no-viz", "--missing-only") or a.startswith("--min-community-size="): + i_arg += 1 + elif a.startswith("--"): + i_arg += 1 + elif watch_path is None: + watch_path = Path(a); i_arg += 1 + else: + i_arg += 1 + if watch_path is None: + watch_path = Path(".") + graph_json = graph_override if graph_override is not None else watch_path / _GRAPHIFY_OUT / "graph.json" + if not graph_json.exists(): + print( + f"error: no graph found at {graph_json} — run /graphify first", + file=sys.stderr, + ) + sys.exit(1) + from networkx.readwrite import json_graph as _jg + from graphify.build import build_from_json + from graphify.cluster import cluster, score_all, remap_communities_to_previous + from graphify.analyze import ( + god_nodes, + surprising_connections, + suggest_questions, + ) + from graphify.report import generate + from graphify.export import to_json, to_html + + stages = _StageTimer(co_timing) + print("Loading existing graph...") + # Solution 3 (#1019): don't hard-exit on an oversized graph.json here. + # Core outputs (graph.json + GRAPH_REPORT.md) still get written; the + # graph.html render below falls back to the community-aggregation view + # (node_limit=5000) when over the cap. + from graphify.security import check_graph_file_size_cap as _check_cap + _over_cap = False + try: + _check_cap(graph_json) + except ValueError: + _over_cap = True + try: + _over_cap_bytes = graph_json.stat().st_size + except OSError: + _over_cap_bytes = -1 + print( + f"warning: graph.json exceeds cap ({_over_cap_bytes} bytes); " + f"falling back to community-aggregation view (node_limit=5000)", + file=sys.stderr, + ) + _raw = json.loads(graph_json.read_text(encoding="utf-8")) + _directed = bool(_raw.get("directed", False)) + G = build_from_json(_raw, directed=_directed) + print(f"Graph: {G.number_of_nodes()} nodes, {G.number_of_edges()} edges") + stages.mark("load") + print("Re-clustering...") + communities = cluster(G, resolution=co_resolution, exclude_hubs_percentile=co_exclude_hubs) + # Mirror the watch/update path (#822): map new cids to prior ones by + # node-overlap so the existing .graphify_labels.json keeps attaching + # to the same conceptual community after re-clustering. Without this, + # labels follow raw cid index and become misaligned whenever the + # graph has changed between labeling and cluster-only (#1027). + previous_node_community = { + n["id"]: n["community"] + for n in _raw.get("nodes", []) + if n.get("community") is not None and n.get("id") is not None + } + if previous_node_community: + communities = remap_communities_to_previous(communities, previous_node_community) + stages.mark("cluster") + cohesion = score_all(G, communities) + gods = god_nodes(G) + surprises = surprising_connections(G, communities) + stages.mark("analyze") + # Where outputs (GRAPH_REPORT.md, re-clustered graph.json, labels, + # analysis, html) land. When `--graph` points at a graph INSIDE a + # graphify-out/ dir (another project/tenant's output), write beside it, + # not into a stray graphify-out/ in the CWD (#1747). But when `--graph` + # points at an arbitrary path — e.g. a `backup/graph.json` archived + # before re-clustering (#934) — fall back to the CWD's graphify-out/, + # which is the restore-into-place workflow that test pins. The default + # (no --graph) case already has graph_json under watch_path/graphify-out. + _out_name = Path(_GRAPHIFY_OUT).name + if graph_override is not None and graph_json.parent.name == _out_name: + out = graph_json.parent + else: + out = watch_path / _GRAPHIFY_OUT + out.mkdir(parents=True, exist_ok=True) + labels_path = out / ".graphify_labels.json" + existing_labels: dict[int, str] = {} + if labels_path.exists(): + try: + existing_labels = { + int(k): v + for k, v in json.loads(labels_path.read_text(encoding="utf-8")).items() + if isinstance(v, str) + } + except Exception: + existing_labels = {} + # Accumulate token usage from the labeling LLM calls so cluster-only mode + # reports real cost instead of a hardcoded zero (#1694). Stays {0, 0} on + # the reuse / no-label paths, which make no LLM calls. + label_token_usage = {"input": 0, "output": 0} + # #2073: a --no-label run produces only "Community N" placeholders. + # Persisting them (plus a matching .sig) made the reuse branch treat them + # as fresh forever, permanently blocking real labeling on later runs. + placeholder_only = False + if labels_path.exists() and not force_relabel: + # Reuse saved labels, but don't blindly trust them: the graph may have + # been re-scoped/re-clustered since labeling, in which case a cid now + # covers a DIFFERENT community and its old (LLM) name is wrong (#label-stale). + # Validate each community against the membership signature saved beside the + # labels; any community that changed (or has no saved label) is renamed by + # its current hub — deterministic and correct-by-construction — and the user + # is told to `graphify label` for fresh LLM names. Unchanged communities keep + # their saved label. When no signature sidecar exists (labels predate this), + # fall back to hub-filling only the communities missing a label. + from graphify.cluster import community_member_sigs, label_communities_by_hub + sig_path = labels_path.parent / (labels_path.name + ".sig") + saved_sigs: dict[int, str] = {} + if sig_path.exists(): + try: + saved_sigs = { + int(k): v for k, v in + json.loads(sig_path.read_text(encoding="utf-8")).items() + if isinstance(v, str) + } + except Exception: + saved_sigs = {} + cur_sigs = community_member_sigs(communities) + count_mismatch = len(existing_labels) != len(communities) + labels = {} + hub_labels: dict[int, str] | None = None + changed = 0 + for cid in communities: + # A persisted "Community {cid}" is a placeholder, not an earned + # label — treat it as absent so the hub labeler replaces it and an + # already-polluted sidecar (e.g. from a prior --no-label run) heals + # instead of suppressing real labels forever (#2073). + have_label = ( + cid in existing_labels + and existing_labels[cid] != f"Community {cid}" + ) + if saved_sigs: + # Precise: the membership signature tells us if this exact + # community changed since it was labeled. + fresh = have_label and saved_sigs.get(cid) == cur_sigs.get(cid) + else: + # No signature sidecar (labels predate it). A differing community + # COUNT means the labels describe a different clustering, so a cid's + # old label can't be trusted; equal count is the best "same" signal. + fresh = have_label and not count_mismatch + if fresh: + labels[cid] = existing_labels[cid] + else: + if hub_labels is None: + hub_labels = label_communities_by_hub(G, communities) + labels[cid] = hub_labels[cid] + if have_label: + changed += 1 + if changed: + print( + f"[graphify] community set changed since labeling " + f"({len(existing_labels)} saved labels, {len(communities)} communities now; " + f"renamed {changed} community(ies) by their hub). " + f"Run `graphify label` to refresh names with the LLM.", + file=sys.stderr, + ) + elif no_label and not force_relabel: + labels = {cid: f"Community {cid}" for cid in communities} + placeholder_only = True + else: + # No labels file yet (or `graphify label` forced a refresh). When run + # standalone there is no orchestrating agent to do skill.md Step 5, so + # auto-name communities rather than leave "Community N" (#1097). + from graphify.cluster import label_communities_by_hub + from graphify.llm import generate_community_labels + print("Labeling communities...") + # Deterministic, LLM-free base labels: name each community after its + # highest-degree hub, so the report is readable even with no backend + # (previously bare "Community N"). A configured LLM backend overrides these + # with richer names below; its no-backend placeholder fallback does NOT. + hub_labels = label_communities_by_hub(G, communities) + label_communities_input = communities + labels = dict(hub_labels) + if missing_only: + labels = { + cid: existing_labels.get(cid, hub_labels[cid]) + for cid in communities + } + label_communities_input = { + cid: members + for cid, members in communities.items() + if cid not in existing_labels or existing_labels.get(cid) == f"Community {cid}" + } + generated_labels, _ = generate_community_labels( + G, label_communities_input, backend=label_backend, model=label_model, gods=gods, + max_concurrency=label_max_concurrency, batch_size=label_batch_size, + usage_out=label_token_usage, + ) + # Only let the LLM OVERRIDE where it produced a real name — its no-backend + # fallback returns "Community {cid}" placeholders, which must not clobber + # the deterministic hub labels. + labels.update({ + cid: v for cid, v in generated_labels.items() + if v and v != f"Community {cid}" + }) + stages.mark("label") + questions = suggest_questions(G, communities, labels) + tokens = label_token_usage + from graphify.export import _git_head as _gh + _commit = _gh() + from graphify.report import load_learning_for_report as _llfr + report = generate(G, communities, cohesion, labels, gods, surprises, + {"warning": "cluster-only mode — file stats not available"}, + tokens, str(watch_path), suggested_questions=questions, + min_community_size=min_community_size, built_at_commit=_commit, + learning=_llfr(out / "graph.json")) + (out / "GRAPH_REPORT.md").write_text(report, encoding="utf-8") + stages.mark("report") + from graphify.export import backup_if_protected as _backup + _backup(out) + analysis = { + "communities": {str(k): v for k, v in communities.items()}, + "cohesion": {str(k): v for k, v in cohesion.items()}, + "gods": gods, + "surprises": surprises, + "questions": questions, + } + (out / ".graphify_analysis.json").write_text( + json.dumps(analysis, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + to_json(G, communities, str(out / "graph.json"), community_labels=labels) + # Don't persist placeholder-only labels (or their .sig): leaving the + # sidecar absent lets a later run generate real labels instead of reading + # back "Community N" as authoritative (#2073). + if not placeholder_only: + from graphify.paths import write_json_atomic as _wja + _wja(labels_path, {str(k): v for k, v in labels.items()}, ensure_ascii=False) + # Membership signatures beside the labels so a later cluster-only can + # detect which communities changed and avoid reusing a stale label + # (see reuse above). + from graphify.cluster import community_member_sigs as _cms + (labels_path.parent / (labels_path.name + ".sig")).write_text( + json.dumps({str(k): v for k, v in _cms(communities).items()}), encoding="utf-8") + + # Mirror watch.py pattern: gate to_html so core outputs (graph.json + + # GRAPH_REPORT.md) always land. Honor --no-viz explicitly; otherwise + # fall back to ValueError handling so an oversized graph doesn't crash + # the CLI mid-write and leave a stale graph.html on disk. + html_target = out / "graph.html" + if no_viz: + if html_target.exists(): + html_target.unlink() + stages.mark("export"); stages.total() + print(f"Done - {len(communities)} communities. GRAPH_REPORT.md and graph.json updated (--no-viz; graph.html removed).") + else: + try: + # Over-cap fallback (#1019): force the community-aggregation + # path so an oversized graph still renders a usable graph.html. + _node_limit = 5000 if _over_cap else None + to_html(G, communities, str(html_target), community_labels=labels or None, + node_limit=_node_limit) + stages.mark("export"); stages.total() + print(f"Done - {len(communities)} communities. GRAPH_REPORT.md, graph.json and graph.html updated.") + except ValueError as viz_err: + if html_target.exists(): + html_target.unlink() + print(f"Skipped graph.html: {viz_err}") + stages.mark("export"); stages.total() + print(f"Done - {len(communities)} communities. GRAPH_REPORT.md and graph.json updated.") + + elif cmd == "update": + force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes") + no_cluster = False + args = sys.argv[2:] + watch_arg: str | None = None + for a in args: + if a == "--force": + force = True + continue + if a == "--no-cluster": + no_cluster = True + continue + if a.startswith("-"): + print(f"error: unknown update option: {a}", file=sys.stderr) + sys.exit(2) + if watch_arg is not None: + print("error: update accepts at most one path argument", file=sys.stderr) + sys.exit(2) + watch_arg = a + + if watch_arg is not None: + watch_path = Path(watch_arg) + else: + # Try to recover the scan root saved by the last full build + saved = Path(_GRAPHIFY_OUT) / ".graphify_root" + if saved.exists(): + watch_path = Path(saved.read_text(encoding="utf-8").strip()) + else: + watch_path = Path(".") + if not watch_path.exists(): + print(f"error: path not found: {watch_path}", file=sys.stderr) + sys.exit(1) + from graphify.watch import _rebuild_code + + print(f"Re-extracting code files in {watch_path} (no LLM needed)...") + # Interactive CLI: block on the per-repo lock rather than skip, so the + # user sees their explicit `graphify update` complete instead of + # exiting silently when a hook-driven rebuild happens to be running. + ok = _rebuild_code(watch_path, force=force, no_cluster=no_cluster, block_on_lock=True) + if ok: + print("Code graph updated. For doc/paper/image changes run /graphify --update in your AI assistant.") + if not ( + os.environ.get("GEMINI_API_KEY") + or os.environ.get("GOOGLE_API_KEY") + or os.environ.get("MOONSHOT_API_KEY") + or os.environ.get("DEEPSEEK_API_KEY") + or os.environ.get("GRAPHIFY_NO_TIPS") + ): + print("Tip: set GEMINI_API_KEY or GOOGLE_API_KEY to use Gemini for semantic extraction.") + else: + print( + "Nothing to update or rebuild failed — check output above.", + file=sys.stderr, + ) + sys.exit(1) + + elif cmd == "hook-check": + # Codex Desktop rejects hookSpecificOutput.additionalContext on PreToolUse. + # Keep this as a cross-platform no-op so installed hooks never break Bash + # tool calls. Graph guidance reaches the agent via AGENTS.md / skill instead. + sys.exit(0) + elif cmd == "hook-guard": + # Shell-agnostic Claude/Codebuddy PreToolUse guard (#522). Replaces the old + # inline-bash hooks that failed on Windows. Prints an additionalContext nudge + # toward graphify when a fresh in-project graph exists; always exits 0. In + # strict mode (opt-in, `hook-guard read --strict`) it blocks the first raw + # read per session via the JSON permissionDecision payload — never via exit + # code — and downgrades to the nudge thereafter. + _run_hook_guard( + sys.argv[2] if len(sys.argv) > 2 else "", + strict="--strict" in sys.argv[3:], + ) + sys.exit(0) + elif cmd == "check-update": + if len(sys.argv) < 3: + print("Usage: graphify check-update ", file=sys.stderr) + sys.exit(1) + from graphify.watch import check_update + + check_update(Path(sys.argv[2]).resolve()) + sys.exit(0) + elif cmd == "tree": + # Emit a D3 v7 collapsible-tree HTML view of graph.json: + # expand-all / collapse-all / reset-view buttons, multi-line + # wrapText labels with separately-coloured name + count, + # depth-based palette, click-to-toggle subtree, hover inspector + # showing top-K outbound edges per symbol. + from typing import Optional as _Opt + from graphify.tree_html import write_tree_html, DEFAULT_MAX_CHILDREN + graph_path = Path(_GRAPHIFY_OUT) / "graph.json" + output_path: "_Opt[Path]" = None + root: "_Opt[str]" = None + max_children = DEFAULT_MAX_CHILDREN + top_k_edges = 0 + project_label: "_Opt[str]" = None + args = sys.argv[2:] + i_arg = 0 + while i_arg < len(args): + a = args[i_arg] + if a == "--graph" and i_arg + 1 < len(args): + graph_path = Path(args[i_arg + 1]); i_arg += 2 + elif a == "--output" and i_arg + 1 < len(args): + output_path = Path(args[i_arg + 1]); i_arg += 2 + elif a == "--root" and i_arg + 1 < len(args): + root = args[i_arg + 1]; i_arg += 2 + elif a == "--max-children" and i_arg + 1 < len(args): + max_children = int(args[i_arg + 1]); i_arg += 2 + elif a == "--top-k-edges" and i_arg + 1 < len(args): + top_k_edges = int(args[i_arg + 1]); i_arg += 2 + elif a == "--label" and i_arg + 1 < len(args): + project_label = args[i_arg + 1]; i_arg += 2 + elif a in ("-h", "--help"): + print("Usage: graphify tree [--graph PATH] [--output HTML]") + print(" --graph PATH path to graph.json (default graphify-out/graph.json)") + print(" --output HTML output path (default graphify-out/GRAPH_TREE.html)") + print(" --root PATH filesystem root (default: longest common dir of all source_files)") + print(" --max-children N cap visible children per node (default 200)") + print(" --top-k-edges N pre-compute top-K outbound edges per symbol (default 12)") + print(" --label NAME project label shown in the page header") + return + else: + i_arg += 1 + if not graph_path.is_file(): + print(f"error: graph.json not found at {graph_path}", file=sys.stderr) + sys.exit(1) + _enforce_graph_size_cap_or_exit(graph_path) + if output_path is None: + output_path = graph_path.parent / "GRAPH_TREE.html" + out = write_tree_html( + graph_path=graph_path, output_path=output_path, + root=root, max_children=max_children, + top_k_edges=top_k_edges, project_label=project_label, + ) + size_kb = out.stat().st_size / 1024 + print(f"wrote {out} ({size_kb:.1f} KB)") + print(f"open with: xdg-open {out} (or file://{out.resolve()})") + sys.exit(0) + + elif cmd == "merge-driver": + # git merge driver for graph.json — takes (base, current, other) and writes + # the union of current+other nodes/edges back to current. Exits 1 on + # corrupt input so git surfaces the conflict instead of silently + # accepting a poisoned merge (see F-005). + # Usage: graphify merge-driver %O %A %B (set in .git/config merge driver) + if len(sys.argv) < 5: + print("Usage: graphify merge-driver ", file=sys.stderr) + sys.exit(1) + _base_path, _current_path, _other_path = sys.argv[2], sys.argv[3], sys.argv[4] + # Hard caps so a malicious or corrupted graph.json cannot exhaust memory + # at parse time. 50 MB / 100k nodes are well above any realistic graph + # (typical graphs are <5 MB / <50k nodes); anything larger should fail + # the merge so a human can investigate. + _MERGE_MAX_BYTES = 50 * 1024 * 1024 + _MERGE_MAX_NODES = 100_000 + import networkx as _nx + from networkx.readwrite import json_graph as _jg + def _load_graph(p: str): + path_obj = Path(p) + try: + size = path_obj.stat().st_size + except OSError as exc: + raise RuntimeError(f"cannot stat {p}: {exc}") from exc + if size > _MERGE_MAX_BYTES: + raise RuntimeError( + f"graph.json {p} is {size} bytes, exceeds {_MERGE_MAX_BYTES}-byte cap" + ) + data = json.loads(path_obj.read_text(encoding="utf-8")) + # A committed raw (--no-cluster) graph stores edges under "edges"; + # parse via the shared links/edges-normalizing loader (#2212). + from graphify.paths import load_node_link_graph as _lnlg + return _lnlg(data), data + try: + G_cur, _ = _load_graph(_current_path) + G_oth, _ = _load_graph(_other_path) + except Exception as exc: + print(f"[graphify merge-driver] error loading graphs: {exc}", file=sys.stderr) + sys.exit(1) # surface the conflict so git doesn't accept a corrupt merge + merged = _nx.compose(G_cur, G_oth) + if merged.number_of_nodes() > _MERGE_MAX_NODES: + print( + f"[graphify merge-driver] merged graph has {merged.number_of_nodes()} nodes, " + f"exceeds {_MERGE_MAX_NODES}-node cap; aborting merge.", + file=sys.stderr, + ) + sys.exit(1) + try: + out_data = _jg.node_link_data(merged, edges="links") + except TypeError: + out_data = _jg.node_link_data(merged) + from graphify.paths import write_json_atomic + write_json_atomic(_current_path, out_data, indent=2) + sys.exit(0) + + elif cmd == "merge-graphs": + # graphify merge-graphs graph1.json graph2.json ... --out merged.json + args = sys.argv[2:] + graph_paths: list[Path] = [] + out_path = Path(_GRAPHIFY_OUT) / "merged-graph.json" + i = 0 + while i < len(args): + if args[i] == "--out" and i + 1 < len(args): + out_path = Path(args[i + 1]) + i += 2 + else: + graph_paths.append(Path(args[i])) + i += 1 + if len(graph_paths) < 2: + print( + "Usage: graphify merge-graphs [...] [--out merged.json]", + file=sys.stderr, + ) + sys.exit(1) + import networkx as _nx + from networkx.readwrite import json_graph as _jg + from graphify.build import prefix_graph_for_global as _prefix, distinct_repo_tags as _repo_tags + graphs = [] + for gp in graph_paths: + if not gp.exists(): + print(f"error: not found: {gp}", file=sys.stderr) + sys.exit(1) + _enforce_graph_size_cap_or_exit(gp) + data = json.loads(gp.read_text(encoding="utf-8")) + # Normalize edges/links key before loading — graphify writes "links" + # via node_link_data but older runs may have used "edges" (#738). + if "links" not in data and "edges" in data: + data = dict(data, links=data["edges"]) + try: + G = _jg.node_link_graph(data, edges="links") + except TypeError: + G = _jg.node_link_graph(data) + graphs.append(G) + # nx.compose requires all graphs to be the same type. When input graphs + # come from different sources (e.g. an AST-only run vs a full LLM run) one + # may be a MultiGraph and another a Graph. Normalise everything to Graph + # (the graphify default) by converting MultiGraphs with nx.Graph(). + def _to_simple(g: "_nx.Graph") -> "_nx.Graph": + # nx.compose requires every graph to be the same type. Inputs may + # disagree on BOTH axes — directed vs undirected, and multi vs simple + # — because per-repo graph.json files are written by different extract + # paths at different times. Normalise everything to a plain undirected + # Graph (the merged cross-repo view is undirected anyway), which covers + # DiGraph / MultiGraph / MultiDiGraph. Without this a directed input + # crashed compose with "All graphs must be directed or undirected" (#1606). + if type(g) is not _nx.Graph: + return _nx.Graph(g) + return g + # Unique repo tag per graph. The bare `graphify-out/..` dir name is not + # unique across inputs (src/graphify-out and frontend/src/graphify-out both + # → "src"), which collides same-stem node ids and silently merges unrelated + # entities (#1729). distinct_repo_tags guarantees a distinct prefix per graph. + repo_tags = _repo_tags(graph_paths) + naive_tags = [gp.parent.parent.name for gp in graph_paths] + if len(set(naive_tags)) != len(naive_tags): + print(f" note: repo dir names collide; using distinct tags: {', '.join(repo_tags)}") + merged = _nx.Graph() + for G, repo_tag in zip(graphs, repo_tags): + prefixed = _to_simple(_prefix(G, repo_tag)) + merged = _nx.compose(merged, prefixed) + try: + out_data = _jg.node_link_data(merged, edges="links") + except TypeError: + out_data = _jg.node_link_data(merged) + out_path.parent.mkdir(parents=True, exist_ok=True) + from graphify.paths import write_json_atomic as _wja + _wja(out_path, out_data, indent=2) + print(f"Merged {len(graphs)} graphs -> {merged.number_of_nodes()} nodes, {merged.number_of_edges()} edges") + print(f"Written to: {out_path}") + + elif cmd == "clone": + if len(sys.argv) < 3: + print( + "Usage: graphify clone [--branch ] [--out ]", + file=sys.stderr, + ) + sys.exit(1) + url = sys.argv[2] + branch: str | None = None + out_dir: Path | None = None + args = sys.argv[3:] + i = 0 + while i < len(args): + if args[i] == "--branch" and i + 1 < len(args): + branch = args[i + 1] + i += 2 + elif args[i] == "--out" and i + 1 < len(args): + out_dir = Path(args[i + 1]) + i += 2 + else: + i += 1 + local_path = _clone_repo(url, branch=branch, out_dir=out_dir) + print(local_path) + + elif cmd == "export": + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + if subcmd not in ("html", "callflow-html", "obsidian", "wiki", "svg", "graphml", "neo4j", "falkordb"): + print("Usage: graphify export ", file=sys.stderr) + print(" html [--graph PATH] [--labels PATH] [--node-limit N] [--no-viz]", file=sys.stderr) + print(" callflow-html [GRAPH|DIR] [--graph PATH] [--labels PATH] [--report PATH] [--sections PATH] [--output HTML]", file=sys.stderr) + print(" [--lang auto|zh-CN|en] [--max-sections N] [--diagram-scale N]", file=sys.stderr) + print(" obsidian [--graph PATH] [--labels PATH] [--dir PATH]", file=sys.stderr) + print(" wiki [--graph PATH] [--labels PATH]", file=sys.stderr) + print(" svg [--graph PATH] [--labels PATH]", file=sys.stderr) + print(" graphml [--graph PATH]", file=sys.stderr) + print(" neo4j [--graph PATH] [--push URI] [--user U] [--password P]", file=sys.stderr) + print(" (or set NEO4J_PASSWORD instead of --password to keep it off argv)", file=sys.stderr) + print(" falkordb [--graph PATH] [--push URI] [--user U] [--password P]", file=sys.stderr) + print(" (or set FALKORDB_PASSWORD instead of --password to keep it off argv)", file=sys.stderr) + sys.exit(1) + + # Parse shared args + args = sys.argv[3:] + graph_path = Path(_GRAPHIFY_OUT) / "graph.json" + graph_path_explicit = False + labels_path = Path(_GRAPHIFY_OUT) / ".graphify_labels.json" + labels_path_explicit = False + report_path = Path(_GRAPHIFY_OUT) / "GRAPH_REPORT.md" + report_path_explicit = False + sections_path: Path | None = None + callflow_output: Path | None = None + callflow_lang = "auto" + callflow_max_sections = 15 + callflow_diagram_scale = 1.0 + callflow_max_diagram_nodes = 18 + callflow_max_diagram_edges = 24 + analysis_path = Path(_GRAPHIFY_OUT) / ".graphify_analysis.json" + node_limit = 5000 + no_viz = False + obsidian_dir = Path(_GRAPHIFY_OUT) / "obsidian" + # Shared push-connection settings for the graph-database sinks (neo4j, + # falkordb), parsed from the generic --push/--user/--password flags below. + push_uri: str | None = None + push_user = "neo4j" # Neo4j default user; FalkorDB auth is optional and ignores it + # F-031: prefer an env var so the password never appears on argv (visible + # in `ps` output / shell history). The explicit --password flag still + # overrides it. Each sink reads its own var: FALKORDB_PASSWORD for falkordb, + # NEO4J_PASSWORD otherwise. + push_password: str | None = ( + os.environ.get("FALKORDB_PASSWORD") if subcmd == "falkordb" + else os.environ.get("NEO4J_PASSWORD") + ) or None + i = 0 + while i < len(args): + a = args[i] + if a == "--graph" and i + 1 < len(args): + graph_path = Path(args[i + 1]) + graph_path_explicit = True + i += 2 + elif a == "--labels" and i + 1 < len(args): + labels_path = Path(args[i + 1]) + labels_path_explicit = True + i += 2 + elif a == "--report" and i + 1 < len(args): + report_path = Path(args[i + 1]) + report_path_explicit = True + i += 2 + elif a == "--sections" and i + 1 < len(args): + sections_path = Path(args[i + 1]); i += 2 + elif a == "--output" and i + 1 < len(args): + callflow_output = Path(args[i + 1]).expanduser() + if not callflow_output.is_absolute(): + callflow_output = Path.cwd() / callflow_output + i += 2 + elif a == "--lang" and i + 1 < len(args): + callflow_lang = args[i + 1]; i += 2 + elif a == "--max-sections" and i + 1 < len(args): + callflow_max_sections = int(args[i + 1]); i += 2 + elif a == "--diagram-scale" and i + 1 < len(args): + callflow_diagram_scale = float(args[i + 1]); i += 2 + elif a == "--max-diagram-nodes" and i + 1 < len(args): + callflow_max_diagram_nodes = int(args[i + 1]); i += 2 + elif a == "--max-diagram-edges" and i + 1 < len(args): + callflow_max_diagram_edges = int(args[i + 1]); i += 2 + elif a in ("-h", "--help") and subcmd == "callflow-html": + print("Usage: graphify export callflow-html [GRAPH|DIR] [--graph PATH] [--labels PATH]") + print(" --report PATH path to GRAPH_REPORT.md") + print(" --sections PATH JSON section definitions") + print(" --output HTML output path (default graphify-out/-callflow.html)") + print(" --lang LANG auto, zh-CN, en, etc. (default auto)") + print(" --max-sections N maximum auto-derived sections (default 15)") + print(" --diagram-scale N Mermaid diagram scale (default 1.0)") + print(" --max-diagram-nodes N representative nodes per section (default 18)") + print(" --max-diagram-edges N representative edges per section (default 24)") + sys.exit(0) + elif a == "--node-limit" and i + 1 < len(args): + node_limit = int(args[i + 1]); i += 2 + elif a == "--no-viz": + no_viz = True; i += 1 + elif a == "--dir" and i + 1 < len(args): + obsidian_dir = Path(args[i + 1]); i += 2 + elif a == "--push" and i + 1 < len(args): + push_uri = args[i + 1]; i += 2 + elif a == "--user" and i + 1 < len(args): + push_user = args[i + 1]; i += 2 + elif a == "--password" and i + 1 < len(args): + push_password = args[i + 1]; i += 2 + elif subcmd == "callflow-html" and not a.startswith("-") and not graph_path_explicit: + candidate = Path(a) + if candidate.name == "graph.json" or candidate.suffix.lower() == ".json": + graph_path = candidate + elif (candidate / "graph.json").exists(): + graph_path = candidate / "graph.json" + else: + graph_path = candidate / _GRAPHIFY_OUT / "graph.json" + graph_path_explicit = True + i += 1 + else: + i += 1 + + graph_path = graph_path.expanduser() + if graph_path_explicit: + graph_out_dir = graph_path.parent + if not labels_path_explicit: + labels_path = graph_out_dir / ".graphify_labels.json" + if not report_path_explicit: + report_path = graph_out_dir / "GRAPH_REPORT.md" + labels_path = labels_path.expanduser() + report_path = report_path.expanduser() + + if not graph_path.exists(): + print(f"error: graph not found: {graph_path}. Run /graphify first.", file=sys.stderr) + sys.exit(1) + + if subcmd == "callflow-html": + from graphify.callflow_html import write_callflow_html as _write_callflow_html + out = _write_callflow_html( + graph=graph_path, + report=report_path, + labels=labels_path, + sections=sections_path, + output=callflow_output, + lang=callflow_lang, + max_sections=callflow_max_sections, + diagram_scale=callflow_diagram_scale, + max_diagram_nodes=callflow_max_diagram_nodes, + max_diagram_edges=callflow_max_diagram_edges, + verbose=True, + ) + print(f"callflow HTML written - open in any browser: {out}") + sys.exit(0) + + from networkx.readwrite import json_graph as _jg + from graphify.build import build_from_json as _bfj + from graphify.security import check_graph_file_size_cap as _check_cap + + # Solution 3 (#1019): for the HTML view, an oversized graph.json should + # not be a hard error. Detect the over-cap condition here and fall back + # to the community-aggregation view (node_limit=5000) below instead of + # exiting 1. All other subcommands keep the hard cap. + _over_cap = False + try: + _check_cap(graph_path) + except ValueError as _cap_err: + if subcmd == "html": + _over_cap = True + try: + _over_cap_bytes = graph_path.stat().st_size + except OSError: + _over_cap_bytes = -1 + print( + f"warning: graph.json exceeds cap ({_over_cap_bytes} bytes); " + f"falling back to community-aggregation view (node_limit=5000)", + file=sys.stderr, + ) + else: + print(f"error: {_cap_err}", file=sys.stderr) + sys.exit(1) + _raw = json.loads(graph_path.read_text(encoding="utf-8")) + if "links" not in _raw and "edges" in _raw: + _raw = dict(_raw, links=_raw["edges"]) + try: + G = _jg.node_link_graph(_raw, edges="links") + except TypeError: + G = _jg.node_link_graph(_raw) + + # Load optional analysis/labels + communities: dict[int, list[str]] = {} + if analysis_path.exists(): + _an = json.loads(analysis_path.read_text(encoding="utf-8")) + communities = {int(k): v for k, v in _an.get("communities", {}).items()} + cohesion: dict[int, float] = {int(k): v for k, v in _an.get("cohesion", {}).items()} + gods_data = _an.get("gods", []) + else: + cohesion = {} + gods_data = [] + + # Fallback: graph.json carries the per-node community as a node attribute + # (`to_json` writes it on every node). The analysis sidecar is the + # canonical source — but the post-commit / watch rebuild path doesn't + # regenerate it, and `extract` may have its temp files cleaned up. When + # that happens, `graphify export html` previously bailed with + # "Single community - aggregated view not useful." even though the + # per-node attribute had the right data all along. Reconstruct from + # the graph itself so downstream subcommands (html, obsidian, wiki, + # svg, graphml, neo4j) don't silently produce a degraded artifact. + if not communities: + reconstructed: dict[int, list[str]] = {} + for node_id, data in G.nodes(data=True): + cid_raw = data.get("community") + if cid_raw is None: + continue + try: + cid = int(cid_raw) + except (TypeError, ValueError): + continue + reconstructed.setdefault(cid, []).append(str(node_id)) + if reconstructed: + communities = reconstructed + + labels: dict[int, str] = {} + if labels_path.exists(): + labels = {int(k): v for k, v in json.loads(labels_path.read_text(encoding="utf-8")).items()} + + out_dir = graph_path.parent + + if subcmd == "html": + from graphify.export import to_html as _to_html + if no_viz: + html_target = out_dir / "graph.html" + if html_target.exists(): + html_target.unlink() + print("--no-viz: skipped graph.html") + else: + # Over-cap fallback (#1019): force the community-aggregation + # path so the oversized graph still renders a usable artifact. + _effective_node_limit = 5000 if _over_cap else node_limit + _to_html(G, communities, str(out_dir / "graph.html"), + community_labels=labels or None, node_limit=_effective_node_limit) + if G.number_of_nodes() <= _effective_node_limit: + print(f"graph.html written - open in any browser, no server needed") + if _over_cap: + sys.exit(0) + + elif subcmd == "obsidian": + from graphify.export import to_obsidian as _to_obsidian, to_canvas as _to_canvas + n = _to_obsidian(G, communities, str(obsidian_dir), + community_labels=labels or None, cohesion=cohesion or None) + print(f"Obsidian vault: {n} notes in {obsidian_dir}/") + _to_canvas(G, communities, str(obsidian_dir / "graph.canvas"), + community_labels=labels or None) + print(f"Canvas: {obsidian_dir}/graph.canvas") + print(f"Open {obsidian_dir}/ as a vault in Obsidian.") + + elif subcmd == "wiki": + from graphify.wiki import to_wiki as _to_wiki + from graphify.analyze import god_nodes as _god_nodes + if not communities: + print( + "error: .graphify_analysis.json is missing or empty — refusing to export wiki to prevent data loss.\n" + "Run `graphify extract .` (or `graphify cluster-only .`) to regenerate community data first.", + file=sys.stderr, + ) + sys.exit(1) + if not gods_data: + gods_data = _god_nodes(G) + n = _to_wiki(G, communities, str(out_dir / "wiki"), + community_labels=labels or None, cohesion=cohesion or None, + god_nodes_data=gods_data) + print(f"Wiki: {n} articles written to {out_dir}/wiki/") + print(f" {out_dir}/wiki/index.md -> agent entry point") + + elif subcmd == "svg": + from graphify.export import to_svg as _to_svg + _to_svg(G, communities, str(out_dir / "graph.svg"), + community_labels=labels or None) + print(f"graph.svg written - embeds in Obsidian, Notion, GitHub READMEs") + + elif subcmd == "graphml": + from graphify.export import to_graphml as _to_graphml + _to_graphml(G, communities, str(out_dir / "graph.graphml")) + print(f"graph.graphml written - open in Gephi, yEd, or any GraphML tool") + + elif subcmd == "neo4j": + if push_uri: + from graphify.export import push_to_neo4j as _push + if push_password is None: + print("error: --password required for --push", file=sys.stderr) + sys.exit(1) + result = _push(G, uri=push_uri, user=push_user, + password=push_password, communities=communities) + print(f"Pushed to Neo4j: {result['nodes']} nodes, {result['edges']} edges") + else: + from graphify.export import to_cypher as _to_cypher + _to_cypher(G, str(out_dir / "cypher.txt")) + print(f"cypher.txt written - import with: cypher-shell < {out_dir}/cypher.txt") + + elif subcmd == "falkordb": + if push_uri: + from graphify.export import push_to_falkordb as _push + result = _push(G, uri=push_uri, user=push_user, + password=push_password, communities=communities) + print(f"Pushed to FalkorDB: {result['nodes']} nodes, {result['edges']} edges") + else: + from graphify.export import to_cypher as _to_cypher + _to_cypher(G, str(out_dir / "cypher.txt")) + print(f"cypher.txt written ({out_dir}/cypher.txt) - statements are OpenCypher. " + f"FalkorDB's GRAPH.QUERY runs one statement at a time (no bulk script " + f"import), so load a graph with: graphify export falkordb --push " + f"falkordb://localhost:6379") + + elif cmd == "benchmark": + from graphify.benchmark import run_benchmark, print_benchmark + + graph_path = sys.argv[2] if len(sys.argv) > 2 else _default_graph_path() + _enforce_graph_size_cap_or_exit(Path(graph_path)) + # Try to load corpus_words from detect output + corpus_words = None + detect_path = Path(".graphify_detect.json") + if detect_path.exists(): + try: + detect_data = json.loads(detect_path.read_text(encoding="utf-8")) + corpus_words = detect_data.get("total_words") + except Exception: + pass + result = run_benchmark(graph_path, corpus_words=corpus_words) + print_benchmark(result) + + elif cmd == "global": + subcmd = sys.argv[2] if len(sys.argv) > 2 else "" + from graphify.global_graph import ( + global_add as _global_add, + global_remove as _global_remove, + global_list as _global_list, + global_path as _global_path, + ) + if subcmd == "add": + # graphify global add [--as ] + args = sys.argv[3:] + source = None + tag = None + i = 0 + while i < len(args): + if args[i] == "--as" and i + 1 < len(args): + tag = args[i + 1]; i += 2 + elif not source: + source = Path(args[i]); i += 1 + else: + i += 1 + if not source: + print("Usage: graphify global add [--as ]", file=sys.stderr) + sys.exit(1) + tag = tag or source.parent.parent.name + try: + result = _global_add(source, tag) + if result["skipped"]: + print(f"'{tag}' unchanged since last add - global graph not modified.") + else: + print(f"Added '{tag}' to global graph: +{result['nodes_added']} nodes, " + f"-{result['nodes_removed']} pruned. Global: {_global_path()}") + except Exception as exc: + print(f"error: {exc}", file=sys.stderr); sys.exit(1) + elif subcmd == "remove": + tag = sys.argv[3] if len(sys.argv) > 3 else "" + if not tag: + print("Usage: graphify global remove ", file=sys.stderr); sys.exit(1) + try: + removed = _global_remove(tag) + print(f"Removed '{tag}' from global graph ({removed} nodes pruned).") + except KeyError as exc: + print(f"error: {exc}", file=sys.stderr); sys.exit(1) + elif subcmd == "list": + repos = _global_list() + if not repos: + print("Global graph is empty. Use 'graphify global add' to add a project.") + else: + print(f"Global graph: {_global_path()}") + for tag, info in repos.items(): + print(f" {tag}: {info.get('node_count', '?')} nodes, added {info.get('added_at', '?')[:10]}") + elif subcmd == "path": + print(_global_path()) + else: + print("Usage: graphify global [add|remove|list|path]", file=sys.stderr); sys.exit(1) + + elif cmd == "extract": + # Headless full-pipeline extraction for CI / scripts (#698). + # Runs detect -> AST extraction on code -> semantic LLM extraction on + # docs/papers/images -> merge -> build -> cluster -> write outputs. + # Unlike the skill.md path (which runs through Claude Code subagents), + # this calls extract_corpus_parallel directly using whichever backend + # has an API key set. + if len(sys.argv) < 3: + print( + "Usage: graphify extract [--backend gemini|kimi|claude|openai|deepseek|ollama] " + "[--model M] [--mode deep] [--out DIR|--output DIR] [--google-workspace] [--no-cluster] " + "[--no-gitignore] [--code-only] " + "[--max-workers N] [--token-budget N] [--max-concurrency N] " + "[--api-timeout S] [--postgres DSN] [--cargo] [--allow-partial] [--timing]", + file=sys.stderr, + ) + sys.exit(1) + + has_path = True + if sys.argv[2].startswith("-"): + has_path = False + target = Path(".").resolve() + else: + target = Path(sys.argv[2]).resolve() + if not target.exists(): + print(f"error: path not found: {target}", file=sys.stderr) + sys.exit(1) + + backend: str | None = None + model: str | None = None + extract_mode: str | None = None + out_dir: Path | None = None + cli_postgres_dsn: str | None = None + cli_cargo: bool = False + cli_allow_partial: bool = False + no_cluster = False + dedup_llm = False + google_workspace = False + global_merge = False + code_only = False + no_gitignore = False + global_repo_tag: str | None = None + # Performance/tuning knobs (issue #792). None means "use library default". + cli_max_workers: int | None = None + cli_token_budget: int | None = None + cli_max_concurrency: int | None = None + cli_api_timeout: float | None = None + # Clustering tuning knobs + cli_resolution: float = 1.0 + cli_exclude_hubs: float | None = None + cli_excludes: list[str] = [] + cli_timing: bool = False + # --force parity with `graphify update`: the flag or GRAPHIFY_FORCE=1 + # disables the incremental gate and skips semantic-cache reads (#1894). + force = os.environ.get("GRAPHIFY_FORCE", "").lower() in ("1", "true", "yes") + + def _parse_int(name: str, raw: str) -> int: + try: + v = int(raw) + except ValueError: + print(f"error: {name} must be a positive integer (got {raw!r})", file=sys.stderr) + sys.exit(2) + if v <= 0: + print(f"error: {name} must be > 0 (got {v})", file=sys.stderr) + sys.exit(2) + return v + + def _parse_float(name: str, raw: str) -> float: + try: + v = float(raw) + except ValueError: + print(f"error: {name} must be a positive number (got {raw!r})", file=sys.stderr) + sys.exit(2) + if v <= 0: + print(f"error: {name} must be > 0 (got {v})", file=sys.stderr) + sys.exit(2) + return v + + args = sys.argv[3:] if has_path else sys.argv[2:] + i = 0 + while i < len(args): + a = args[i] + if a == "--backend" and i + 1 < len(args): + backend = args[i + 1]; i += 2 + elif a.startswith("--backend="): + backend = a.split("=", 1)[1]; i += 1 + elif a == "--model" and i + 1 < len(args): + model = args[i + 1]; i += 2 + elif a.startswith("--model="): + model = a.split("=", 1)[1]; i += 1 + elif a == "--mode" and i + 1 < len(args): + extract_mode = args[i + 1]; i += 2 + elif a.startswith("--mode="): + extract_mode = a.split("=", 1)[1]; i += 1 + elif a in ("--out", "--output") and i + 1 < len(args): + # --output is an alias of --out (#2004): it was silently dropped + # before, and `graphify tree` already documents --output, so the + # mistake is natural. (--output= does not startswith --out=.) + out_dir = Path(args[i + 1]); i += 2 + elif a.startswith(("--out=", "--output=")): + out_dir = Path(a.split("=", 1)[1]); i += 1 + elif a == "--no-cluster": + no_cluster = True; i += 1 + elif a == "--dedup-llm": + dedup_llm = True; i += 1 + elif a == "--code-only": + code_only = True; i += 1 + elif a == "--google-workspace": + google_workspace = True; i += 1 + elif a == "--no-gitignore": + no_gitignore = True; i += 1 + elif a == "--global": + global_merge = True; i += 1 + elif a == "--as" and i + 1 < len(args): + global_repo_tag = args[i + 1]; i += 2 + elif a == "--max-workers" and i + 1 < len(args): + cli_max_workers = _parse_int("--max-workers", args[i + 1]); i += 2 + elif a.startswith("--max-workers="): + cli_max_workers = _parse_int("--max-workers", a.split("=", 1)[1]); i += 1 + elif a == "--token-budget" and i + 1 < len(args): + cli_token_budget = _parse_int("--token-budget", args[i + 1]); i += 2 + elif a.startswith("--token-budget="): + cli_token_budget = _parse_int("--token-budget", a.split("=", 1)[1]); i += 1 + elif a == "--max-concurrency" and i + 1 < len(args): + cli_max_concurrency = _parse_int("--max-concurrency", args[i + 1]); i += 2 + elif a.startswith("--max-concurrency="): + cli_max_concurrency = _parse_int("--max-concurrency", a.split("=", 1)[1]); i += 1 + elif a == "--api-timeout" and i + 1 < len(args): + cli_api_timeout = _parse_float("--api-timeout", args[i + 1]); i += 2 + elif a.startswith("--api-timeout="): + cli_api_timeout = _parse_float("--api-timeout", a.split("=", 1)[1]); i += 1 + elif a == "--resolution" and i + 1 < len(args): + cli_resolution = _parse_float("--resolution", args[i + 1]); i += 2 + elif a.startswith("--resolution="): + cli_resolution = _parse_float("--resolution", a.split("=", 1)[1]); i += 1 + elif a == "--exclude-hubs" and i + 1 < len(args): + cli_exclude_hubs = float(args[i + 1]); i += 2 + elif a.startswith("--exclude-hubs="): + cli_exclude_hubs = float(a.split("=", 1)[1]); i += 1 + elif a == "--exclude" and i + 1 < len(args): + cli_excludes.append(args[i + 1]); i += 2 + elif a.startswith("--exclude="): + cli_excludes.append(a.split("=", 1)[1]); i += 1 + elif a == "--postgres" and i + 1 < len(args): + cli_postgres_dsn = args[i + 1]; i += 2 + elif a.startswith("--postgres="): + cli_postgres_dsn = a.split("=", 1)[1]; i += 1 + elif a == "--cargo": + cli_cargo = True + i += 1 + elif a == "--force": + force = True; i += 1 + elif a == "--allow-partial": + cli_allow_partial = True; i += 1 + elif a == "--timing": + cli_timing = True; i += 1 + else: + i += 1 + + if not has_path and cli_postgres_dsn is None: + print("error: must specify a path to scan or a --postgres DSN", file=sys.stderr) + sys.exit(1) + + _VALID_MODES = {"deep"} + if extract_mode is not None and extract_mode not in _VALID_MODES: + print( + f"error: unknown --mode '{extract_mode}'. " + f"Available: {', '.join(sorted(_VALID_MODES))}", + file=sys.stderr, + ) + sys.exit(2) + deep_mode = extract_mode == "deep" + if deep_mode: + print("[graphify extract] deep mode enabled: richer semantic extraction") + + # CLI flag wins over env var. Setting GRAPHIFY_API_TIMEOUT here so + # _call_openai_compat picks it up without needing a new kwarg path. + if cli_api_timeout is not None: + os.environ["GRAPHIFY_API_TIMEOUT"] = str(cli_api_timeout) + if cli_max_workers is not None: + os.environ["GRAPHIFY_MAX_WORKERS"] = str(cli_max_workers) + + # Resolve output dir. The user-facing contract is "/graphify-out/" + # so a fresh checkout writes graphify-out/ at the project root, matching + # the skill.md pipeline. + out_root = (out_dir.resolve() if out_dir else target) + graphify_out = out_root / _GRAPHIFY_OUT + graphify_out.mkdir(parents=True, exist_ok=True) + # Persist corpus-shaping options so later update/watch/hook rebuilds + # use the same file set as the initial extraction (#1886). + from graphify.watch import ( + _write_build_config as _write_build_cfg, + _read_build_excludes as _read_build_ex, + _read_build_gitignore as _read_build_gi, + ) + # #1971 persistence: an explicit --no-gitignore persists False; a later + # flag-less `graphify extract` must NOT clobber it back to True, which + # would make the git-ignored code silently disappear again (the exact + # complaint #1971 is about). Honor the persisted value for THIS run when + # the flag is absent (read before the write below), and write False only + # when the flag is set — None leaves the setting as-is, mirroring how + # #1886 persists --exclude. + _effective_gitignore = False if no_gitignore else _read_build_gi(graphify_out) + # An explicit list replaces the persisted one; omission reuses it. + _effective_excludes = cli_excludes or _read_build_ex(graphify_out) + _write_build_cfg( + graphify_out, + excludes=cli_excludes or None, + gitignore=False if no_gitignore else None, + ) + + stages = _StageTimer(cli_timing) + + from graphify.detect import ( + detect as _detect, + detect_incremental as _detect_incremental, + save_manifest as _save_manifest, + ) + manifest_path = graphify_out / "manifest.json" + existing_graph_path = graphify_out / "graph.json" + # #1925: a missing manifest.json must not degrade to a full scan that + # discards the existing graph's semantic layer. An existing graph.json + # is a sufficient incremental baseline: detect_incremental treats an + # absent manifest as "everything is new" (re-extract all, nothing + # deleted), and build_merge + _stale_graph_sources reconcile replaced + # and genuinely-deleted sources against the current corpus, so doc/ + # paper/image nodes survive a --code-only rebuild instead of being + # dropped with the rest of the committed graph. + incremental_mode = existing_graph_path.exists() if has_path else False + # --force: full scan, not the manifest-gated incremental diff — a warm + # unchanged tree would otherwise dispatch zero files (#1894). + incremental_mode = incremental_mode and not force + if force: + print("[graphify extract] --force: full re-scan, semantic cache reads skipped") + elif incremental_mode and not manifest_path.exists(): + print( + "[graphify extract] manifest.json missing; using existing " + "graph.json as the incremental baseline (all files re-checked; " + "nodes for files outside this run's scope are preserved)" + ) + + if not has_path: + detection = {} + code_files = [] + doc_files = [] + paper_files = [] + image_files = [] + deleted_files = [] + excluded_files = [] + graph_stale_sources = [] + unchanged_total = 0 + files_by_type = {} + elif incremental_mode: + print(f"[graphify extract] incremental scan of {target}") + detection = _detect_incremental( + target, + manifest_path=str(manifest_path), + google_workspace=google_workspace or None, + extra_excludes=_effective_excludes or None, + gitignore=_effective_gitignore, + ) + files_by_type = detection.get("files", {}) + new_by_type = detection.get("new_files", {}) + code_files = [Path(p) for p in new_by_type.get("code", [])] + doc_files = [Path(p) for p in new_by_type.get("document", [])] + paper_files = [Path(p) for p in new_by_type.get("paper", [])] + image_files = [Path(p) for p in new_by_type.get("image", [])] + deleted_files = list(detection.get("deleted_files", [])) + excluded_files = list(detection.get("excluded_files", [])) + unchanged_total = sum(len(v) for v in detection.get("unchanged_files", {}).values()) + # #1909: derive the prune set from the existing graph itself, not + # just the manifest. A file that became excluded without ever + # being manifest-listed (every pre-#1897 graph is in this state) + # still has stale nodes carried forward by build_merge unless the + # graph's own sources are reconciled against the current corpus. + _seen_files = {f for _fl in files_by_type.values() for f in _fl} + _seen_files.update(detection.get("unclassified", [])) + graph_stale_sources = _stale_graph_sources( + existing_graph_path, target, _seen_files, detection=detection + ) + else: + print(f"[graphify extract] scanning {target}") + detection = _detect( + target, + google_workspace=google_workspace or None, + extra_excludes=_effective_excludes or None, + cache_root=out_root, + gitignore=_effective_gitignore, + ) + files_by_type = detection.get("files", {}) + code_files = [Path(p) for p in files_by_type.get("code", [])] + doc_files = [Path(p) for p in files_by_type.get("document", [])] + paper_files = [Path(p) for p in files_by_type.get("paper", [])] + image_files = [Path(p) for p in files_by_type.get("image", [])] + deleted_files = [] + excluded_files = [] + graph_stale_sources = [] + unchanged_total = 0 + + semantic_files = doc_files + paper_files + image_files + # --code-only: index code (pure local AST, no key) and skip the semantic + # (doc/paper/image) pass entirely, so a mixed repo doesn't hard-fail when no + # LLM backend is configured (#1734). Report what was skipped rather than + # silently dropping it. + if code_only and semantic_files: + print( + f"[graphify extract] --code-only: skipping {len(semantic_files)} " + f"non-code file(s) ({len(doc_files)} docs, {len(paper_files)} papers, " + f"{len(image_files)} images) — no LLM extraction" + ) + semantic_files = [] + doc_files = [] + paper_files = [] + image_files = [] + if deep_mode and incremental_mode and not code_only: + # Deep mode reads/writes its own cache namespace + # (cache/semantic-deep/), so the manifest's changed-file gate is + # not a valid proxy for deep coverage: over a warm unchanged tree + # it dispatches zero files and `--mode deep` silently no-ops + # (#1894). Widen the semantic pass to the FULL live + # doc/paper/image set (``files_by_type`` from detect_incremental, + # which already excludes excluded files) and let the + # mode-namespaced cache decide hits/misses — the first deep run + # re-dispatches everything (deep namespace cold), later deep runs + # hit the deep cache. + _deep_all = [ + Path(p) + for _ftype in ("document", "paper", "image") + for p in files_by_type.get(_ftype, []) + ] + if len(_deep_all) != len(semantic_files): + print( + f"[graphify extract] deep mode: widening semantic pass from " + f"{len(semantic_files)} changed to {len(_deep_all)} live " + f"doc/paper/image file(s); the deep semantic cache decides " + f"what is re-extracted" + ) + semantic_files = _deep_all + if incremental_mode: + # Excluded-but-alive files are reported separately from deletions + # (#1908): they still exist on disk, the scan just stopped + # covering them (ignore rules / --exclude changed). + _excl_note = f"; {len(excluded_files)} excluded" if excluded_files else "" + print( + f"[graphify extract] {len(code_files)} code, {len(doc_files)} docs, " + f"{len(paper_files)} papers, {len(image_files)} images changed; " + f"{unchanged_total} unchanged; {len(deleted_files)} deleted" + f"{_excl_note}" + ) + else: + print( + f"[graphify extract] found {len(code_files)} code, " + f"{len(doc_files)} docs, {len(paper_files)} papers, " + f"{len(image_files)} images" + ) + # Surface files that were seen but not classified (extensionless non-shebang + # project files like Dockerfile/Makefile, or unsupported extensions), so they + # are no longer invisible in graphify's own output (#1692). + _unclassified = detection.get("unclassified", []) if isinstance(detection, dict) else [] + if _unclassified: + _names = ", ".join(sorted({Path(p).name for p in _unclassified})[:6]) + _more = f" (+{len(_unclassified) - 6} more)" if len(_unclassified) > 6 else "" + print( + f"[graphify extract] {len(_unclassified)} file(s) not classified " + f"(no supported extension or shebang), skipped: {_names}{_more}" + ) + # Name the files dropped by the sensitive-file filter so a wrongly-flagged + # source/doc is visible, not just a count (#2106). Operational skips + # (symlink/office/Workspace) carry a " [reason]" suffix; exclude those here + # so this line reports only the security-heuristic drops. + _sensitive = detection.get("skipped_sensitive", []) if isinstance(detection, dict) else [] + _sec = [s for s in _sensitive if " [" not in s] + if _sec: + _snames = ", ".join(sorted({Path(p).name for p in _sec})[:6]) + _smore = f" (+{len(_sec) - 6} more)" if len(_sec) > 6 else "" + print( + f"[graphify extract] {len(_sec)} file(s) skipped as potentially sensitive " + f"(rename or move if wrongly flagged): {_snames}{_smore}" + ) + stages.mark("detect") + + # Resolve the LLM backend only now that we know whether the corpus + # needs one. A code-only corpus is pure local AST and must not require + # an API key; the key is enforced below only when there's LLM work. + from graphify.llm import ( + BACKENDS as _BACKENDS, + detect_backend as _detect_backend, + estimate_cost as _estimate_cost, + extract_corpus_parallel as _extract_corpus_parallel, + _format_backend_env_keys, + _get_backend_api_key, + ) + needs_llm = bool(semantic_files) or dedup_llm + if backend is None and needs_llm: + backend = _detect_backend() + if backend is not None and backend not in _BACKENDS: + print( + f"error: unknown backend '{backend}'. " + f"Available: {', '.join(sorted(_BACKENDS))}", + file=sys.stderr, + ) + sys.exit(1) + if needs_llm: + if backend is None: + reasons = [] + if semantic_files: + reasons.append( + f"{len(semantic_files)} doc/paper/image file(s) need semantic extraction" + ) + if dedup_llm: + reasons.append("--dedup-llm was passed") + hint = "" + if semantic_files: + hint = (" Or pass --code-only to index just the code " + "(local AST, no key) and skip the non-code files.") + print( + "error: no LLM API key found (" + "; ".join(reasons) + "). " + "Set GEMINI_API_KEY or GOOGLE_API_KEY (gemini), MOONSHOT_API_KEY " + "(kimi), ANTHROPIC_API_KEY (claude), OPENAI_API_KEY (openai), " + "DEEPSEEK_API_KEY (deepseek), or pass --backend. A code-only " + "corpus needs no key." + hint, + file=sys.stderr, + ) + sys.exit(1) + if backend == "ollama": + from graphify.llm import _validate_ollama_base_url + _oll_url = os.environ.get("OLLAMA_BASE_URL", _BACKENDS["ollama"].get("base_url", "")) + try: + _validate_ollama_base_url(_oll_url, warn=False) + except ValueError as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(2) + if not _get_backend_api_key(backend): + allow_no_key = False + if backend == "ollama": + from urllib.parse import urlparse + ollama_url = os.environ.get( + "OLLAMA_BASE_URL", + _BACKENDS["ollama"].get("base_url", ""), + ) + try: + host = (urlparse(ollama_url).hostname or "").lower() + except Exception: + host = "" + allow_no_key = ( + host in ("localhost", "127.0.0.1", "::1") + or host.startswith("127.") + ) + elif backend == "bedrock": + allow_no_key = bool( + os.environ.get("AWS_PROFILE") + or os.environ.get("AWS_REGION") + or os.environ.get("AWS_DEFAULT_REGION") + or os.environ.get("AWS_ACCESS_KEY_ID") + ) + elif backend == "claude-cli": + import shutil as _shutil + allow_no_key = _shutil.which("claude") is not None + if not allow_no_key: + print( + "error: backend 'claude-cli' requires the `claude` CLI on $PATH " + "(install Claude Code and run `claude` once to authenticate).", + file=sys.stderr, + ) + sys.exit(1) + if not allow_no_key: + print( + f"error: backend '{backend}' requires {_format_backend_env_keys(backend)} to be set.", + file=sys.stderr, + ) + sys.exit(1) + + # Track whether this run's extraction was incomplete (a whole extractor + # pass crashed, or some semantic chunks failed). A partial result must not + # be force-written over a good complete graph — the final write falls back + # to the #479 shrink guard unless --allow-partial is set. + _extraction_incomplete = False + # A walk that couldn't fully enumerate the corpus (permission-denied + # subtree, I/O error) yields a legitimately smaller graph that must not + # be force-written over a complete one — same failure class as a crashed + # pass. detect()/detect_incremental() already record these; consume them. + if detection.get("walk_errors"): + _extraction_incomplete = True + + # AST extraction on code files. Empty code list (docs-only corpus) is + # the issue #698 case — skip cleanly instead of crashing inside extract(). + ast_result: dict = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} + if code_files: + from graphify.extract import extract as _ast_extract + # Anchor the cache at the output root, not the scanned project: + # with --out, a /graphify-out/cache/ would leak a + # graphify-out/ dir into a project that asked for external output. + # `root` stays the scanned project so source_file/ids relativize + # against it; conflating the two basenamed every node (#1941). + ast_kwargs: dict = {"cache_root": out_root, "root": target} + if cli_max_workers is not None: + ast_kwargs["max_workers"] = cli_max_workers + print(f"[graphify extract] AST extraction on {len(code_files)} code files...") + try: + ast_result = _ast_extract(code_files, **ast_kwargs) + except Exception as exc: + print(f"[graphify extract] AST extraction failed: {exc}", file=sys.stderr) + ast_result = {"nodes": [], "edges": [], "input_tokens": 0, "output_tokens": 0} + _extraction_incomplete = True # the whole AST pass was lost + stages.mark("AST extract") + + # Semantic extraction on docs/papers/images. Check cache first. + from graphify.cache import ( + check_semantic_cache as _check_semantic_cache, + prune_semantic_cache as _prune_semantic_cache, + save_semantic_cache as _save_semantic_cache, + ) + sem_result: dict = { + "nodes": [], "edges": [], "hyperedges": [], + "input_tokens": 0, "output_tokens": 0, + } + # Semantic files whose extraction truncated this run. They are left + # unstamped in the manifest so detect_incremental re-queues them next run + # (mirrors the #933 failed-chunk handling); captured below before the + # _partial markers are stripped from the corpus. + _partial_semantic_files: set[str] = set() + sem_cache_hits = 0 + sem_cache_misses = 0 + # Deep mode uses its own namespace (cache/semantic-deep/) so deep and + # standard results for the same content never shadow each other (#1894). + sem_cache_mode = "deep" if deep_mode else None + # Entries are attributed to the extraction prompt that produced them, so + # a release that changes the prompt re-extracts rather than replaying the + # older vintage alongside the new one (#1939). Read and write must pass + # the same prompt, or the write lands where the next read won't look. + from graphify.llm import _extraction_system as _sem_prompt_for + sem_prompt = _sem_prompt_for(deep=deep_mode) + if semantic_files: + sem_paths_str = [str(p) for p in semantic_files] + if force: + # --force: skip the cache READ so every semantic file is + # re-dispatched; the save below still runs so the fresh + # results replace the stale entries. + cached_nodes, cached_edges, cached_hyperedges = [], [], [] + uncached_paths = list(sem_paths_str) + else: + cached_nodes, cached_edges, cached_hyperedges, uncached_paths = ( + _check_semantic_cache(sem_paths_str, root=target, cache_root=out_root, + mode=sem_cache_mode, prompt=sem_prompt) + ) + sem_cache_hits = len(semantic_files) - len(uncached_paths) + sem_cache_misses = len(uncached_paths) + sem_result["nodes"].extend(cached_nodes) + sem_result["edges"].extend(cached_edges) + sem_result["hyperedges"].extend(cached_hyperedges) + if sem_cache_hits: + print(f"[graphify extract] semantic cache: {sem_cache_hits} hit / {sem_cache_misses} miss") + + if uncached_paths: + print(f"[graphify extract] semantic extraction on {len(uncached_paths)} files via {backend}...") + corpus_kwargs: dict = { + "backend": backend, + "model": model, + "root": target, + "cache_root": out_root, + } + if deep_mode: + corpus_kwargs["deep_mode"] = True + if cli_token_budget is not None: + corpus_kwargs["token_budget"] = cli_token_budget + if cli_max_concurrency is not None: + corpus_kwargs["max_concurrency"] = cli_max_concurrency + + # Minimal progress callback so the CLI is no longer silent + # during long local-inference runs (issue #792 addendum). + # Also track per-chunk success so we can fail loudly when + # every chunk errors (e.g. missing backend SDK package). + _chunk_stats = {"total": 0, "succeeded": 0} + def _progress(idx: int, total: int, _result: dict) -> None: + _chunk_stats["total"] = total + _chunk_stats["succeeded"] += 1 + print( + f"[graphify extract] chunk {idx + 1}/{total} done", + flush=True, + ) + corpus_kwargs["on_chunk_done"] = _progress + + try: + fresh = _extract_corpus_parallel( + [Path(p) for p in uncached_paths], + **corpus_kwargs, + ) + except ImportError as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + except Exception as exc: + print( + f"[graphify extract] semantic extraction failed: {exc}", + file=sys.stderr, + ) + fresh = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} + _extraction_incomplete = True # the semantic pass crashed + + # on_chunk_done only fires after a chunk succeeds. If fresh + # semantic extraction was requested and no chunks completed, + # fail instead of writing an AST-only graph with exit 0. + if uncached_paths and _chunk_stats["succeeded"] == 0: + print( + f"[graphify extract] error: all semantic chunks failed " + f"for backend '{backend}' ({len(uncached_paths)} uncached files) - " + f"see per-chunk errors above. If you see 'requires the X package', " + f"run `pip install X` and retry.", + file=sys.stderr, + ) + sys.exit(1) + # Some (but not all) chunks failed — the graph is missing nodes + # from the failed chunks, so it must not clobber a larger complete + # graph without an explicit --allow-partial override. + if _chunk_stats["total"] and _chunk_stats["succeeded"] < _chunk_stats["total"]: + _extraction_incomplete = True + # Which files truncated this run (item markers + the empty-parse + # _partial_files set). Computed BEFORE the save so it can be passed + # as partial_source_files: without it, a file whose only truncated + # chunk parsed empty (so it has no item markers here) would be + # written as a complete cache entry, re-promoting it (#1950). + from graphify.llm import ( + _partial_source_files as _partial_sf, + _strip_partial_markers as _strip_partial, + ) + _partial_semantic_files = set(_partial_sf(fresh)) + try: + _save_semantic_cache( + fresh.get("nodes", []), + fresh.get("edges", []), + fresh.get("hyperedges", []), + root=target, + cache_root=out_root, + allowed_source_files=uncached_paths, + mode=sem_cache_mode, + prompt=sem_prompt, + partial_source_files=_partial_semantic_files or None, + ) + except Exception as exc: + print(f"[graphify extract] warning: could not write semantic cache: {exc}", file=sys.stderr) + # Strip the markers before the corpus feeds the graph so the + # internal flag never leaks into graph.json. + _strip_partial(fresh) + sem_result["nodes"].extend(fresh.get("nodes", [])) + sem_result["edges"].extend(fresh.get("edges", [])) + sem_result["hyperedges"].extend(fresh.get("hyperedges", [])) + sem_result["input_tokens"] += fresh.get("input_tokens", 0) + sem_result["output_tokens"] += fresh.get("output_tokens", 0) + + # Prune orphaned semantic cache entries. The semantic cache is + # content-hash-keyed and unversioned, so it is never swept by the AST + # version-cleanup: every content change or file deletion leaves a + # permanent orphan that accumulates unbounded (#1527). Sweep it against + # the FULL live document set (``files_by_type`` — present in both the + # incremental and full branches), NOT the incremental ``semantic_files`` + # changed-subset, which would delete every unchanged doc's valid entry. + # Best-effort: a prune failure must never break extraction. + # Hash keys are anchored to the corpus (``target``) — the same anchor + # the cache read/write above use — while the stat-index artifact + # follows the cache location (``out_root``). Anchoring these hashes to + # ``out_root`` instead would mismatch every key under ``--out`` and + # sweep the entire fresh cache as orphaned (#1990/#1991). + try: + from graphify.cache import file_hash as _file_hash + _live_hashes: set[str] = set() + for _kind in ("document", "paper", "image"): + for _fp in files_by_type.get(_kind, []): + _abs = Path(_fp) + if not _abs.is_absolute(): + _abs = Path(target) / _abs + if not _abs.is_file(): + continue # deleted/missing — leave out so its entry is pruned + try: + _live_hashes.add(_file_hash(_abs, target, cache_root=out_root)) + except OSError: + pass + # A pathless database extraction has no filesystem corpus to sweep. + if has_path: + _prune_semantic_cache(out_root, _live_hashes) + except Exception as exc: + print(f"[graphify extract] warning: could not prune semantic cache: {exc}", file=sys.stderr) + stages.mark("semantic extract") + + pg_result: dict = {"nodes": [], "edges": []} + if cli_postgres_dsn is not None: + from graphify.pg_introspect import introspect_postgres + print(f"[graphify extract] introspecting PostgreSQL schema...") + try: + pg_result = introspect_postgres(cli_postgres_dsn) + except (ConnectionError, ImportError) as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + print(f"[graphify extract] PostgreSQL: {len(pg_result['nodes'])} nodes, " + f"{len(pg_result['edges'])} edges") + + cargo_result: dict = {"nodes": [], "edges": []} + if cli_cargo: + from graphify.cargo_introspect import introspect_cargo + print("[graphify extract] introspecting Cargo workspace...") + try: + cargo_result = introspect_cargo(target) + except (ConnectionError, ImportError, OSError) as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + print(f"[graphify extract] Cargo: {len(cargo_result['nodes'])} nodes, " + f"{len(cargo_result['edges'])} edges") + + # Merge AST + semantic + pg_result + cargo_result. Order matters for deduplication: passing AST + # first means semantic node attributes win on collision (richer labels + # for symbols also referenced in docs). Hyperedges only come from the + # semantic side. + merged: dict = { + "nodes": list(ast_result.get("nodes", [])) + list(sem_result.get("nodes", [])) + list(pg_result.get("nodes", [])) + list(cargo_result.get("nodes", [])), + "edges": list(ast_result.get("edges", [])) + list(sem_result.get("edges", [])) + list(pg_result.get("edges", [])) + list(cargo_result.get("edges", [])), + "hyperedges": list(sem_result.get("hyperedges", [])), + "input_tokens": ast_result.get("input_tokens", 0) + sem_result.get("input_tokens", 0), + "output_tokens": ast_result.get("output_tokens", 0) + sem_result.get("output_tokens", 0), + } + + graph_json_path = graphify_out / "graph.json" + analysis_path = graphify_out / ".graphify_analysis.json" + + # Build a manifest-safe files dict: only stamp semantic_hash for files + # that actually produced output (cache hit or fresh extraction). Files + # whose chunk failed have no source_file entry in sem_result — leaving + # their semantic_hash empty so detect_incremental re-queues them (#933). + # Path normalization against the scan root happens inside the helper + # (#1897) so fresh root-relative source_files match detect()'s + # absolute file lists. + _manifest_files = _stamped_manifest_files(files_by_type, sem_result, target, + partial_source_files=_partial_semantic_files) + + # Files dispatched this run but dropped by _stamped_manifest_files + # above (failed chunk, LLM omission, or any future exclusion) still + # carry a stale semantic_hash from a prior successful run in the + # on-disk manifest; save_manifest's seed loop would otherwise copy it + # verbatim and mask the omission (#1948). Derived from semantic_files + # — what was actually SENT to the backend this run (narrowed by the + # incremental gate and --code-only, widened by deep mode) — NOT from + # files_by_type: the full live corpus includes untouched files that + # were never dispatched, and clearing those would blank the whole + # manifest on every partial incremental run, forcing a full-corpus + # re-extraction on the next one. + _stamped_semantic = { + f for _flist in _manifest_files.values() for f in _flist + } + _cleared_semantic = {str(p) for p in semantic_files} - _stamped_semantic + + # Full-scan manifest saves prune rows for in-root files that left the + # scan corpus but still exist on disk (#1908). The corpus must be the + # RAW detect output (files_by_type), NOT the #933-stamp-filtered + # _manifest_files above — pruning to the filtered set would erase + # failed-chunk/omitted-doc rows and every doc row on --code-only runs. + _scan_corpus = ( + {f for _fl in files_by_type.values() for f in _fl} + if has_path else None + ) + + def _invalidate_file_manifest_for_db_graph() -> None: + if has_path: + return + try: + manifest_path.unlink(missing_ok=True) + except OSError as exc: + print(f"error: could not invalidate file manifest: {exc}", file=sys.stderr) + sys.exit(1) + + if no_cluster: + # --no-cluster: dump the raw merged extraction as graph.json. + # No NetworkX, no community detection, no analysis sidecar. + # Dedupe nodes (by id) and parallel edges so the raw output matches the + # clustered path (whose DiGraph collapses both) and stays deterministic + # across modes (#1317; node dedup also collapses shared Swift module + # anchors emitted per importing file, #1327). + from graphify.build import dedupe_edges as _dedupe_edges, dedupe_nodes as _dedupe_nodes + from graphify.export import ( + backup_if_protected as _backup, + existing_graph_node_count as _existing_graph_node_count, + ) + if ( + incremental_mode + and not code_files + and not semantic_files + and not deleted_files + and not pg_result.get("nodes") + and not pg_result.get("edges") + and not cargo_result.get("nodes") + and not cargo_result.get("edges") + ): + # An exclusion-only change reaches this gate (excluded files + # are deliberately NOT in deleted_files, #1908) but must still + # scrub the newly-excluded sources from the raw graph (#1909). + # This path never runs build_merge, so prune in place. + if graph_stale_sources: + _n_pruned = _prune_graph_json_sources( + existing_graph_path, graph_stale_sources + ) + if _n_pruned: + print( + f"[graphify extract] pruned {_n_pruned} node(s) from " + f"{len(graph_stale_sources)} source file(s) no longer " + "in the scan (deleted or excluded)." + ) + print( + "[graphify extract] no incremental changes detected " + "(--no-cluster); outputs left untouched." + ) + try: + _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus, clear_semantic=_cleared_semantic) + except Exception as exc: + print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) + stages.total() + sys.exit(0) + + if incremental_mode: + # #2169: this raw path used to write ONLY this run's extraction + # over graph.json — on an incremental run that is just the + # changed files, silently dropping every node/edge owned by an + # unchanged file. Merge the existing graph forward first, with + # the same replace/prune semantics as the clustered path's + # build_merge: re-extracted sources replaced, deleted + + # excluded + graph-stale sources pruned, everything else + # carried. Survivors are prepended, so the dedupe below keeps + # this run's fresh attributes for re-extracted nodes. + from graphify.build import merge_raw_extraction as _merge_raw_extraction + _raw_prune_sources: list[str] = list(deleted_files) + for _src in list(excluded_files) + graph_stale_sources: + if _src not in _raw_prune_sources: + _raw_prune_sources.append(_src) + try: + merged = _merge_raw_extraction( + merged, + graph_path=existing_graph_path, + prune_sources=_raw_prune_sources or None, + root=target, + ) + except RuntimeError as exc: + # Existing graph present but unparseable: refuse to + # raw-dump this run's partial extraction over it. + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + merged["nodes"] = _dedupe_nodes(merged["nodes"]) + merged["edges"] = _dedupe_edges(merged["edges"]) + # Disambiguate colliding-basename file-node labels (#2032). This raw + # --no-cluster path bypasses build_from_json (where the clustered path + # gets this), so apply it directly on the merged node list. + from graphify.build import disambiguate_file_labels_in_nodes as _disamb_labels + _disamb_labels(merged["nodes"]) + # Backfill source_file from endpoint nodes — this raw path bypasses + # build_from_json's backfill, and semantic edges sometimes omit it (#1279). + _node_sf = {n.get("id"): n.get("source_file") for n in merged["nodes"]} + for _e in merged["edges"]: + if not _e.get("source_file"): + _e["source_file"] = ( + _node_sf.get(_e.get("source")) or _node_sf.get(_e.get("target")) or "" + ) + # RT-parity for the raw path: an incomplete build must not force a + # partial graph over a larger complete one here either. The clustered + # path gets this from to_json's #479 guard; this path never calls + # to_json, so replicate the shrink check against the existing file and + # exit before the write/manifest unless --allow-partial is set. + if _extraction_incomplete and not cli_allow_partial: + from graphify.export import MALFORMED_GRAPH as _MALFORMED_GRAPH + _existing_n = _existing_graph_node_count(graph_json_path) + _malformed = _existing_n is _MALFORMED_GRAPH + _shrinks = isinstance(_existing_n, int) and len(merged["nodes"]) < _existing_n + if _malformed or _shrinks: + _detail = ( + f"the existing {graph_json_path} is present but unparseable " + "(corrupt or a mid-write), so a shrink cannot be ruled out" + if _malformed + else f"smaller than the existing {graph_json_path} " + f"({len(merged['nodes'])} < {_existing_n} nodes)" + ) + print( + "[graphify extract] error: extraction was incomplete (an AST/" + f"semantic pass failed) and the resulting --no-cluster graph is {_detail}. " + "Refusing to overwrite a complete graph with a partial one. Re-run after " + "fixing the failures, or pass --allow-partial to overwrite anyway.", + file=sys.stderr, + ) + sys.exit(1) + _backup(graphify_out) + _invalidate_file_manifest_for_db_graph() + from graphify.paths import write_json_atomic as _write_json_atomic + _write_json_atomic(graph_json_path, merged, indent=2) + try: + # Record the scan root so a later build_merge / update runbook can + # relativize deleted-file paths correctly even for a custom --out + # (its grandparent-of-graph.json fallback points at the wrong dir + # otherwise, and deleted files never prune — #2012/#1571). + (graphify_out / ".graphify_root").write_text( + str(Path(target).resolve()), encoding="utf-8" + ) + except OSError: + pass + stages.mark("write") + cost = _estimate_cost( + backend, merged["input_tokens"], merged["output_tokens"] + ) + print( + f"[graphify extract] wrote {graph_json_path} — " + f"{len(merged['nodes'])} nodes, {len(merged['edges'])} edges " + f"(no clustering)" + ) + if merged["input_tokens"] or merged["output_tokens"]: + print( + f"[graphify extract] tokens: " + f"{merged['input_tokens']:,} in / " + f"{merged['output_tokens']:,} out, " + f"est. cost: ${cost:.4f}" + ) + try: + if has_path: + _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus, clear_semantic=_cleared_semantic) + except Exception as exc: + print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) + if global_merge: + from graphify.global_graph import global_add as _global_add + _tag = global_repo_tag or target.name + try: + result = _global_add(graphify_out / "graph.json", _tag) + if result["skipped"]: + print(f"[graphify global] '{_tag}' unchanged since last add - skipped.") + else: + print(f"[graphify global] '{_tag}' merged into global graph " + f"(+{result['nodes_added']} nodes, -{result['nodes_removed']} pruned).") + except Exception as exc: + print(f"[graphify global] warning: failed to merge into global graph: {exc}", file=sys.stderr) + stages.total() + sys.exit(0) + + # Build graph + cluster + score + write. + from graphify.build import ( + build as _build, + build_from_json as _build_from_json, + build_merge as _build_merge, + ) + from graphify.cluster import cluster as _cluster, score_all as _score_all + from graphify.export import to_json as _to_json + from graphify.analyze import god_nodes as _god_nodes, surprising_connections as _surprising + dedup_backend = backend if dedup_llm else None + if incremental_mode: + # Prune everything the current scan no longer covers: genuinely + # deleted manifest rows, excluded-but-alive manifest rows (#1908), + # and the graph's own stale sources — which catches files that + # became excluded without ever being manifest-listed (#1909). + _prune_sources: list[str] = list(deleted_files) + for _src in list(excluded_files) + graph_stale_sources: + if _src not in _prune_sources: + _prune_sources.append(_src) + G = _build_merge( + [merged], + graph_path=existing_graph_path, + prune_sources=_prune_sources or None, + dedup=True, + dedup_llm_backend=dedup_backend, + root=target, + ) + else: + G = _build([merged], dedup=True, dedup_llm_backend=dedup_backend, root=target) + stages.mark("build") + if G.number_of_nodes() == 0: + print( + "[graphify extract] graph is empty — extraction produced no nodes. " + "Possible causes: all files skipped, binary-only corpus, or LLM " + "returned no edges.", + file=sys.stderr, + ) + sys.exit(1) + + communities = _cluster(G, resolution=cli_resolution, exclude_hubs_percentile=cli_exclude_hubs) + stages.mark("cluster") + cohesion = _score_all(G, communities) + try: + gods = _god_nodes(G) + except Exception: + gods = [] + try: + surprises = _surprising(G, communities) + except Exception: + surprises = [] + stages.mark("analyze") + + from graphify.export import backup_if_protected as _backup + _backup(graphify_out) + _invalidate_file_manifest_for_db_graph() + # force=True bypasses the #479 shrink guard entirely. A full build + # legitimately shrinks (fuzzy dedup collapse, deleted code) so it keeps + # force=True — EXCEPT when this run's extraction was incomplete (an + # extractor pass crashed or some semantic chunks failed). Then a partial + # graph could silently overwrite a good complete one, so fall back to the + # shrink guard (force=False) unless the user opts in with --allow-partial. + # + # Both write paths are guarded: the clustered path here via to_json's + # #479 check, and the `--no-cluster` raw-dump path above via the same + # shrink check against the existing file (existing_graph_node_count). + # + # Trade-off: this reuses to_json's coarse node-count guard, not the + # source-aware _check_shrink that watch/update use. On an incremental run + # a legitimate deletion that coincides with an unrelated transient chunk + # failure can therefore be refused here — recoverable by re-running or + # passing --allow-partial (the good graph is preserved and the manifest + # is not stamped, so the retry re-extracts). + _force_write = cli_allow_partial or not _extraction_incomplete + _wrote = _to_json(G, communities, str(graph_json_path), force=_force_write) + if not _wrote: + # The shrink guard refused: this partial build is smaller than the + # existing graph. Exit before writing the manifest/marker below, which + # would otherwise stamp these files as done and make the next + # incremental run skip re-extracting them (poisoning the manifest + # against the graph we declined to write). Exit non-zero so a retry + # re-attempts. + print( + "[graphify extract] error: extraction was incomplete (an AST/semantic " + f"pass failed) and the resulting graph is smaller than the existing " + f"{graph_json_path}. Refusing to overwrite a complete graph with a " + "partial one. Re-run after fixing the failures, or pass --allow-partial " + "to overwrite anyway.", + file=sys.stderr, + ) + sys.exit(1) + try: + # See the --no-cluster path above: persist the scan root so build_merge + # can relativize deleted-file paths under a custom --out (#2012/#1571). + (graphify_out / ".graphify_root").write_text( + str(Path(target).resolve()), encoding="utf-8" + ) + except OSError: + pass + stages.mark("export") + if merged.get("output_tokens", 0) > 0: + (graphify_out / ".graphify_semantic_marker").write_text( + json.dumps({"output_tokens": merged["output_tokens"]}), encoding="utf-8" + ) + if global_merge: + from graphify.global_graph import global_add as _global_add + _tag = global_repo_tag or target.name + try: + result = _global_add(graphify_out / "graph.json", _tag) + if result["skipped"]: + print(f"[graphify global] '{_tag}' unchanged since last add - skipped.") + else: + print(f"[graphify global] '{_tag}' merged into global graph " + f"(+{result['nodes_added']} nodes, -{result['nodes_removed']} pruned).") + except Exception as exc: + print(f"[graphify global] warning: failed to merge into global graph: {exc}", file=sys.stderr) + analysis = { + "communities": {str(k): v for k, v in communities.items()}, + "cohesion": {str(k): v for k, v in cohesion.items()}, + "gods": gods, + "surprises": surprises, + "tokens": { + "input": merged["input_tokens"], + "output": merged["output_tokens"], + }, + } + from graphify.paths import write_json_atomic as _wja + _wja(analysis_path, analysis, indent=2) + try: + if has_path: + _save_manifest(_manifest_files, manifest_path=str(manifest_path), kind="both", root=target, scan_corpus=_scan_corpus, clear_semantic=_cleared_semantic) + except Exception as exc: + print(f"[graphify extract] warning: could not write manifest: {exc}", file=sys.stderr) + + cost = _estimate_cost(backend, merged["input_tokens"], merged["output_tokens"]) + print( + f"[graphify extract] wrote {graph_json_path}: " + f"{G.number_of_nodes()} nodes, {G.number_of_edges()} edges, " + f"{len(communities)} communities" + ) + print(f"[graphify extract] wrote {analysis_path}") + if incremental_mode: + _excl_note = f", {len(excluded_files)} excluded" if excluded_files else "" + print( + f"[graphify extract] incremental summary: " + f"{sem_cache_hits + unchanged_total} files cached/unchanged, " + f"{len(code_files) + sem_cache_misses} re-extracted, " + f"{len(deleted_files)} deleted{_excl_note}" + ) + elif sem_cache_hits: + print(f"[graphify extract] semantic cache: {sem_cache_hits} cached, {sem_cache_misses} re-extracted") + if merged["input_tokens"] or merged["output_tokens"]: + print( + f"[graphify extract] tokens: " + f"{merged['input_tokens']:,} in / " + f"{merged['output_tokens']:,} out, " + f"est. cost (~{backend}): ${cost:.4f}" + ) + # extract intentionally stops at graph.json + analysis; the report and + # community labels are produced by `cluster-only` (or an agent's Step 5). + # Point standalone users at it so communities get named (#1097). + print( + "[graphify extract] next: run " + f"`graphify cluster-only {graphify_out.parent}` " + "to generate GRAPH_REPORT.md and name communities" + ) + stages.total() + + elif cmd == "cache-check": + # graphify cache-check [--root ] [--mode | --deep] + # [--prompt-file ] + # Reads file paths (one per line) from , checks semantic cache. + # --mode deep (or --deep) checks the cache/semantic-deep/ namespace + # written by `extract --mode deep` instead of cache/semantic/ (#1894). + # --prompt-file names the extraction prompt the caller will use (an agent's + # references/extraction-spec.md), restricting hits to entries produced by + # that same prompt (#1939). Omitting it reads the unattributed layout, which + # cannot see entries a fingerprinted run wrote. + # Writes: + # graphify-out/.graphify_cached.json — already-cached nodes/edges/hyperedges + # graphify-out/.graphify_uncached.txt — paths that need extraction + # Stdout: "Cache: N hit, M miss" + from graphify.cache import check_semantic_cache + if len(sys.argv) < 3: + print("Usage: graphify cache-check [--root ] " + "[--mode | --deep] [--prompt-file ]", file=sys.stderr) + sys.exit(1) + files_from = Path(sys.argv[2]) + root = Path(".") + cache_mode: str | None = None + prompt_file: str | None = None + i = 3 + while i < len(sys.argv): + if sys.argv[i] == "--root" and i + 1 < len(sys.argv): + root = Path(sys.argv[i + 1]) + i += 2 + elif sys.argv[i] == "--mode" and i + 1 < len(sys.argv): + cache_mode = sys.argv[i + 1] + i += 2 + elif sys.argv[i].startswith("--mode="): + cache_mode = sys.argv[i].split("=", 1)[1] + i += 1 + elif sys.argv[i] == "--deep": + cache_mode = "deep" + i += 1 + elif sys.argv[i] == "--prompt-file" and i + 1 < len(sys.argv): + prompt_file = sys.argv[i + 1] + i += 2 + elif sys.argv[i].startswith("--prompt-file="): + prompt_file = sys.argv[i].split("=", 1)[1] + i += 1 + else: + i += 1 + files = [f for f in files_from.read_text(encoding="utf-8").splitlines() if f.strip()] + cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache( + files, root, mode=cache_mode, prompt_file=prompt_file + ) + out = root / _GRAPHIFY_OUT + out.mkdir(parents=True, exist_ok=True) + if cached_nodes or cached_edges or cached_hyperedges: + (out / ".graphify_cached.json").write_text( + json.dumps({"nodes": cached_nodes, "edges": cached_edges, "hyperedges": cached_hyperedges}, + ensure_ascii=False), + encoding="utf-8", + ) + (out / ".graphify_uncached.txt").write_text("\n".join(uncached), encoding="utf-8") + print(f"Cache: {len(files) - len(uncached)} hit, {len(uncached)} miss") + + elif cmd == "merge-chunks": + # graphify merge-chunks --out + # Concatenates .graphify_chunk_*.json files written by semantic subagents. + # Deduplicates nodes by id (first writer wins). Sums token counts. + import glob as _glob + if len(sys.argv) < 3: + print("Usage: graphify merge-chunks --out ", file=sys.stderr) + sys.exit(1) + out_path: Path | None = None + chunk_args: list[str] = [] + i = 2 + while i < len(sys.argv): + if sys.argv[i] == "--out" and i + 1 < len(sys.argv): + out_path = Path(sys.argv[i + 1]) + i += 2 + else: + chunk_args.append(sys.argv[i]) + i += 1 + if not out_path: + print("error: --out required", file=sys.stderr) + sys.exit(1) + chunk_files: list[str] = [] + for arg in chunk_args: + expanded = _glob.glob(arg) + chunk_files.extend(sorted(expanded) if expanded else [arg]) + merged: dict = {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0} + seen_ids: set[str] = set() + valid_chunks = 0 + # These chunk files are untrusted subagent output. load_validated_... + # stats the file size BEFORE reading it (so a multi-GB chunk can't blow up + # memory), parses the JSON, and validates the security caps + the node/ + # edge id charset that blocks path traversal (#825) — the same enforcement + # the skill merge path applies. A bad chunk is skipped with a warning + # while valid siblings still merge; if every chunk is invalid, fail + # closed instead of reporting success and replacing --out with an empty + # semantic layer. Deliberately NOT wired into + # build_from_json/load_graph_json, which must keep loading valid + # pre-existing graphs. file_type is left to build's coercion (#840). + from graphify.semantic_cleanup import load_validated_semantic_fragment + for cf in chunk_files: + chunk, _chunk_errs = load_validated_semantic_fragment(Path(cf)) + if _chunk_errs: + print( + f"[graphify merge-chunks] warning: skipping invalid chunk {cf}: " + f"{'; '.join(_chunk_errs[:3])}", + file=sys.stderr, + ) + continue + valid_chunks += 1 + for n in chunk.get("nodes", []): + if n.get("id") not in seen_ids: + seen_ids.add(n["id"]) + merged["nodes"].append(n) + merged["edges"].extend(chunk.get("edges", [])) + merged["hyperedges"].extend(chunk.get("hyperedges", [])) + # Coerce token counts: a chunk is untrusted, so a non-numeric + # input_tokens/output_tokens must not abort the whole merge with a + # TypeError after other chunks already merged. + for _tok in ("input_tokens", "output_tokens"): + _v = chunk.get(_tok, 0) + merged[_tok] += _v if isinstance(_v, (int, float)) else 0 + if not valid_chunks: + print( + f"[graphify merge-chunks] error: no valid chunks to merge; " + f"refusing to write {out_path}", + file=sys.stderr, + ) + sys.exit(1) + out_path.parent.mkdir(parents=True, exist_ok=True) + from graphify.paths import write_json_atomic as _wja + _wja(out_path, merged, ensure_ascii=False) + chunk_summary = ( + f"{valid_chunks} chunks" + if valid_chunks == len(chunk_files) + else f"{valid_chunks} of {len(chunk_files)} chunks" + ) + print( + f"Merged {chunk_summary}: {len(merged['nodes'])} nodes, {len(merged['edges'])} edges, " + f"{merged['input_tokens']:,} in / {merged['output_tokens']:,} out tokens" + ) + + elif cmd == "merge-semantic": + # graphify merge-semantic --cached --new --out + # Merges cached semantic results with freshly-extracted chunk results. + # Deduplicates nodes by id (cached entries take priority over new ones). + if len(sys.argv) < 3: + print("Usage: graphify merge-semantic --cached --new --out ", file=sys.stderr) + sys.exit(1) + cached_path: Path | None = None + new_path: Path | None = None + out_path2: Path | None = None + i = 2 + while i < len(sys.argv): + if sys.argv[i] == "--cached" and i + 1 < len(sys.argv): + cached_path = Path(sys.argv[i + 1]); i += 2 + elif sys.argv[i] == "--new" and i + 1 < len(sys.argv): + new_path = Path(sys.argv[i + 1]); i += 2 + elif sys.argv[i] == "--out" and i + 1 < len(sys.argv): + out_path2 = Path(sys.argv[i + 1]); i += 2 + else: + i += 1 + if not out_path2: + print("error: --out required", file=sys.stderr) + sys.exit(1) + empty: dict = {"nodes": [], "edges": [], "hyperedges": []} + cached_data = json.loads(cached_path.read_text(encoding="utf-8")) if cached_path and cached_path.exists() else empty + new_data = json.loads(new_path.read_text(encoding="utf-8")) if new_path and new_path.exists() else empty + seen_ids2: set[str] = set() + all_nodes: list[dict] = [] + for n in cached_data.get("nodes", []) + new_data.get("nodes", []): + if n.get("id") not in seen_ids2: + seen_ids2.add(n["id"]) + all_nodes.append(n) + merged2 = { + "nodes": all_nodes, + "edges": cached_data.get("edges", []) + new_data.get("edges", []), + "hyperedges": cached_data.get("hyperedges", []) + new_data.get("hyperedges", []), + } + out_path2.parent.mkdir(parents=True, exist_ok=True) + from graphify.paths import write_json_atomic as _wja + _wja(out_path2, merged2, ensure_ascii=False) + print(f"Merged: {len(merged2['nodes'])} nodes, {len(merged2['edges'])} edges") + + elif Path(cmd).exists() or cmd in (".", "..") or cmd.startswith(("./", "../", "/", "~")): + # User ran `graphify ` directly — treat as `graphify extract `. + # Common when following the PowerShell note in README (`graphify .`) or + # copy-pasting skill invocations without the leading slash. + sys.argv.insert(2, sys.argv[1]) + sys.argv[1] = "extract" + _reenter_main() + else: + print(f"error: unknown command '{cmd}'", file=sys.stderr) + print("Run 'graphify --help' for usage.", file=sys.stderr) + sys.exit(1) diff --git a/graphify/dedup.py b/graphify/dedup.py index e0ddd2e3b..bc2f1c418 100644 --- a/graphify/dedup.py +++ b/graphify/dedup.py @@ -187,6 +187,134 @@ def _is_code(node: dict) -> bool: return node.get("file_type") == "code" +# ── ID collisions ───────────────────────────────────────────────────────────── + +_ID_SEGMENT = re.compile(r"[^a-z0-9]+") +_EXTENSION = re.compile(r"\.[^./]+$") + + +def _id_prefixes(source_file: str) -> set[str]: + """The ID prefixes a node extracted from ``source_file`` may legitimately mint. + + An ID is ``_``, where the path is the extension-stripped source + path, each segment slugified and joined with ``_``. Every trailing slice of the + path counts as a prefix: the stored path may be absolute or repo-relative, and + graphs built under the pre-#1504 scheme keyed off the bare filename stem. + """ + stem = _EXTENSION.sub("", source_file.replace("\\", "/")) + segments = [s for s in (_ID_SEGMENT.sub("_", p.casefold()).strip("_") + for p in stem.split("/")) if s] + return {"_".join(segments[i:]) for i in range(len(segments))} + + +def _defines_id(node: dict) -> bool: + """True when the node's own source_file is the file its ID encodes. + + A doc that *references* an entity mints the ID of the entity's own file, not one + derived from the doc's path — so the referencing node collides with the defining + node by construction. This separates the two: the definer owns the ID. + """ + nid = node.get("id") or "" + source_file = node.get("source_file") or "" + if not nid or not source_file: + return False + # `nid == prefix` covers a bare file-level node whose id is exactly the + # slugified path with no `_entity` suffix (a semantic node for the file + # itself); `startswith(prefix + "_")` covers the usual `_` id. + return any(nid == prefix or nid.startswith(f"{prefix}_") + for prefix in _id_prefixes(source_file)) + + +def _collision_rank(node: dict) -> tuple: + """A total order for choosing the survivor of an ID collision, independent of + the order the colliding nodes arrive in. + + The winner is the node with the SMALLEST rank. A node whose ``source_file`` + defines the ID always outranks a mere reference; among equally-(non-)defining + nodes it prefers the shorter, more canonical label over a longer qualified + variant, then breaks any remaining tie lexically on label and then source_file + (so the lexically-first path wins) — fully deterministic regardless of order. + """ + label = node.get("label") or "" + return ( + not _defines_id(node), # definers (False) sort before references (True) + len(label), # shorter, more canonical label first + label, # lexical tiebreak + node.get("source_file") or "", # lexically-first source path wins + ) + + +def _same_source_entity(survivor: dict, duplicate: dict) -> bool: + """True when exact-ID records came from the same source file. + + Exact IDs can also collide across files through references or slugged-path + ambiguity (#1504). Keep those records isolated rather than importing + attributes whose provenance belongs to another file. + """ + keep_file = survivor.get("source_file") or "" + lose_file = duplicate.get("source_file") or "" + # Require a non-empty source_file: two provenance-less records ("" == "") + # are NOT proof of the same symbol (#1178), and merging their attributes + # would be a cross-pollination bug in the opposite direction (#2091 review). + return bool(keep_file) and keep_file == lose_file + + +def _merge_missing_attributes(survivor: dict, duplicate: dict) -> dict: + """Fill the survivor's absent/None attributes from a same-source duplicate, + without overriding values the survivor already has (#2091).""" + merged = dict(survivor) + for key, value in duplicate.items(): + # Never inherit a provenance tag from a dropped record: a false + # _origin="ast" on an LLM survivor is read as an authority signal by the + # ghost-merge (#2068) and watch deletion logic (#2091 review). + if key == "_origin": + continue + if value is None: + continue + # Treat an explicit None on the survivor as absent — the codebase emits + # `source_location: None`, and that is exactly the attribute #2091 loses. + if merged.get(key) is None: + merged[key] = value + return merged + + +def _report_id_collision(nid: str, survivor: dict, losers: list[dict]) -> None: + """Report an ID collision in proportion to what dropping the loser actually costs. + + Cross-reference to a defining node: the structural entity and its edges survive; + foreign-file attributes stay isolated, so no collision warning is needed. Same + file, different labels: the extractor emitted two labels for one entity and one is + discarded — note it. Two files that both encode this ID: they are distinct entities + and one is genuinely lost — warn, and point at the extraction split that keeps them + apart (#1504). + """ + keep_file = survivor.get("source_file") or "" + keep_label = survivor.get("label") or "" + for loser in losers: + lose_file = loser.get("source_file") or "" + lose_label = loser.get("label") or "" + if lose_file == keep_file: + if _norm(lose_label) != _norm(keep_label): + print( + f"[graphify] note: node '{nid}' was extracted twice from " + f"'{keep_file}' under different labels — keeping '{keep_label}', " + f"dropping '{lose_label}'.", + file=sys.stderr, + ) + elif _defines_id(survivor) and not _defines_id(loser): + continue # the loser only references the entity the survivor defines + else: + print( + f"[graphify] WARNING: node '{nid}' is minted by two different files — " + f"keeping '{keep_label}' from '{keep_file}', dropping '{lose_label}' " + f"from '{lose_file}'. An ID is derived from the source path plus the " + f"entity name, so this one does not identify a single entity and the " + f"dropped node is lost. To keep them distinct, run 'graphify extract' " + f"per subfolder and merge with 'graphify merge-graphs'.", + file=sys.stderr, + ) + + # ── main entry point ────────────────────────────────────────────────────────── def deduplicate_entities( @@ -220,29 +348,48 @@ def deduplicate_entities( if len(nodes) <= 1: return nodes, edges - # Pre-deduplicate: keep first occurrence of each id. - # Warn when two nodes share an ID but originate from different source files — - # this indicates a cross-chunk ID collision (#1504) where silent data loss occurs. + # Pre-deduplicate: one node per ID. The survivor is the node that *defines* the + # ID (its source_file is the file the ID encodes), not merely the first seen — + # otherwise chunk order decides whether an entity keeps its own attributes or a + # passing cross-reference's. Missing attributes from same-source records are + # retained so AST structure and semantic enrichment can coexist (#2091). + # Genuine cross-file ID collisions stay isolated and are reported below (#1504). seen_ids: dict[str, dict] = {} + dropped: dict[str, list[dict]] = defaultdict(list) for node in nodes: nid = node.get("id", "") if not nid: continue - if nid not in seen_ids: + incumbent = seen_ids.get(nid) + if incumbent is None: + seen_ids[nid] = node + elif _collision_rank(node) < _collision_rank(incumbent): + # Smallest-ranked node wins; the min over a total order is independent + # of the order nodes arrive in, so the survivor no longer depends on + # chunk ordering (#1851). seen_ids[nid] = node + dropped[nid].append(incumbent) else: - existing_sf = seen_ids[nid].get("source_file") or "" - new_sf = node.get("source_file") or "" - if existing_sf != new_sf: - print( - f"[graphify] WARNING: node '{nid}' from '{new_sf}' collides with " - f"node from '{existing_sf}' — the second node will be dropped. " - f"This is a cross-chunk ID collision caused by two files with the " - f"same name in different directories. To avoid data loss, run " - f"'graphify extract' per subfolder and merge with " - f"'graphify merge-graphs'.", - file=sys.stderr, - ) + dropped[nid].append(node) + + # Gap-fill each survivor from its SAME-SOURCE losers, applied in deterministic + # _collision_rank order (best loser first). Merging here — not incrementally in + # the loop above — keeps the merged attributes independent of chunk arrival + # order with 3+ colliding records, preserving the #1851 order-independence + # contract (#2091 review). + for nid, losers in dropped.items(): + survivor = seen_ids[nid] + same_source = sorted( + (l for l in losers if _same_source_entity(survivor, l)), + key=_collision_rank, + ) + for loser in same_source: + survivor = _merge_missing_attributes(survivor, loser) + seen_ids[nid] = survivor + + for nid, losers in dropped.items(): + _report_id_collision(nid, seen_ids[nid], losers) + unique_nodes = list(seen_ids.values()) if len(unique_nodes) <= 1: @@ -264,8 +411,10 @@ def deduplicate_entities( for key, group in norm_to_nodes.items(): if len(group) <= 1: continue - # Partition by source_file — only merge within the same file in Pass 1. - # Cross-file matches fall through to Pass 2 fuzzy matching. + # Partition by source_file — same-file exact matches always merge here. + # Cross-file exact matches are handled just below, gated to `concept` + # nodes only: Pass 2 cannot form them because its candidate list keeps a + # single node per normalized label (#2182). by_file: dict[str, list[dict]] = defaultdict(list) for node in group: sf = node.get("source_file") or "" @@ -280,6 +429,26 @@ def deduplicate_entities( for node in file_group: uf.union(winner["id"], node["id"]) exact_merges += len(file_group) - 1 + # Cross-file residue: union exact matches across files, but only where + # it is provably safe (#2182). `concept` is the one file_type meant to + # unify across files (#1284) — code is keyed by ID (#1205), rationale/ + # document are file-anchored (#1284), and image/paper labels are often + # shared basenames (logo.png). Provenance is required (#1178), and the + # entropy gate mirrors Pass 2 so short generic labels ("API") stay + # distinct. Sorting by id keeps the winner order-independent. + mergeable = sorted( + (n for n in group + if n.get("file_type") == "concept" + and (n.get("source_file") or "") + and _entropy(n.get("label", "")) >= _ENTROPY_THRESHOLD), + key=lambda n: n["id"], + ) + if len(mergeable) > 1: + winner = _pick_winner(mergeable) + for node in mergeable: + if uf.find(winner["id"]) != uf.find(node["id"]): + uf.union(winner["id"], node["id"]) + exact_merges += 1 # ── pass 2: MinHash/LSH + Jaro-Winkler (high-entropy nodes only) ───────── candidates: list[dict] = [] @@ -375,10 +544,14 @@ def deduplicate_entities( score += _COMMUNITY_BOOST if score >= _MERGE_THRESHOLD: - # Identical labels across different source files almost always - # means same-named-but-different symbols (trait impls, wrapper - # methods, common type names). Mirror Pass 1's source_file - # partition for this sub-case. (#1046, leaks #895's fix) + # Belt-and-braces (#1046, narrowed by #2182): candidates are + # norm-unique (`seen_norms` above), so two candidates can + # never share a normalized label and this branch is + # unreachable today. Retained in case candidate selection + # changes. Equal-norm cross-file pairs are handled in Pass 1 + # instead, gated to `concept` nodes — the original #1046 + # rationale (same-named code symbols) was obsoleted by code + # being excluded from label matching entirely (#1205, #1247). if norm_label == neighbor_norm: sf_a = node.get("source_file") or "" sf_b = neighbor.get("source_file") or "" @@ -417,11 +590,16 @@ def deduplicate_entities( total = len(remap) msg = f"[graphify] Deduplicated {total} node(s)" + # Both counters are reported when non-zero. Previous form nested the fuzzy + # branch inside `if exact_merges`, silently dropping the fuzzy count on + # doc/semantic-heavy runs where Pass 1 finds nothing (#1857). + parts: list[str] = [] if exact_merges: - msg += f" ({exact_merges} exact" - if fuzzy_merges: - msg += f", {fuzzy_merges} fuzzy" - msg += ")" + parts.append(f"{exact_merges} exact") + if fuzzy_merges: + parts.append(f"{fuzzy_merges} fuzzy") + if parts: + msg += f" ({', '.join(parts)})" print(msg + ".", flush=True) deduped_nodes = [n for n in unique_nodes if n["id"] not in remap] diff --git a/graphify/detect.py b/graphify/detect.py index 080dab813..d243641e2 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -7,6 +7,7 @@ import shlex from concurrent.futures import ThreadPoolExecutor from enum import Enum +from functools import lru_cache from pathlib import Path from graphify.google_workspace import ( @@ -27,8 +28,8 @@ class FileType(str, Enum): _MANIFEST_PATH = str(out_path("manifest.json")) -CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} -DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.txt', '.rst', '.html', '.yaml', '.yml'} +CODE_EXTENSIONS = {'.py', '.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.mjs', '.cjs', '.ejs', '.ets', '.go', '.rs', '.java', '.groovy', '.gradle', '.cpp', '.cc', '.cxx', '.c', '.h', '.hpp', '.cu', '.cuh', '.metal', '.rb', '.rake', '.swift', '.kt', '.kts', '.cs', '.scala', '.php', '.lua', '.luau', '.toc', '.zig', '.ps1', '.psm1', '.psd1', '.ex', '.exs', '.m', '.mm', '.jl', '.vue', '.svelte', '.astro', '.dart', '.v', '.sv', '.svh', '.sql', '.r', '.f', '.F', '.f90', '.F90', '.f95', '.F95', '.f03', '.F03', '.f08', '.F08', '.pas', '.pp', '.dpr', '.dpk', '.lpr', '.inc', '.dfm', '.lfm', '.lpk', '.sh', '.bash', '.json', '.tf', '.tfvars', '.hcl', '.dm', '.dme', '.dmi', '.dmm', '.dmf', '.sln', '.slnx', '.csproj', '.fsproj', '.vbproj', '.xaml', '.razor', '.cshtml', '.cls', '.trigger'} +DOC_EXTENSIONS = {'.md', '.mdx', '.qmd', '.skill', '.txt', '.rst', '.html', '.yaml', '.yml'} PAPER_EXTENSIONS = {'.pdf'} IMAGE_EXTENSIONS = {'.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg'} OFFICE_EXTENSIONS = {'.docx', '.xlsx'} @@ -93,11 +94,21 @@ def _zip_within_caps(path: Path) -> bool: return False return True -# Parent directories whose contents are always sensitive. -# Checked against path.parts[:-1] (parents only) so a root-level file named -# "credentials" or "secrets" is not falsely flagged by this stage. -_SENSITIVE_DIRS = frozenset({ - ".ssh", ".gnupg", ".aws", ".gcloud", "secrets", ".secrets", "credentials", +# Dedicated credential-store directories: everything beneath them is sensitive, +# with no carve-out — a .py inside ~/.ssh or ~/.aws is tooling for key material, +# not a source package, and keys there are routinely extensionless. +# Both sets are checked against path.parts[:-1] (parents only) so a root-level +# file named "credentials" or "secrets" is not falsely flagged by this stage. +_CREDENTIAL_STORE_DIRS = frozenset({ + ".ssh", ".gnupg", ".aws", ".gcloud", +}) + +# Bare-name directories that are as often legitimate source packages (Go +# internal/secrets, a credentials/ service module) as credential stores. Their +# contents are sensitive EXCEPT genuine programming-language source, mirroring +# the Stage 3 keyword carve-out (#1666) at the directory level (#1943). +_AMBIGUOUS_SENSITIVE_DIRS = frozenset({ + "secrets", ".secrets", "credentials", }) # Files that may contain secrets - skip silently. These patterns are specific @@ -105,9 +116,17 @@ def _zip_within_caps(path: Path) -> bool: _SENSITIVE_PATTERNS = [ re.compile(r'(^|[\\/])\.(env|envrc)(\.|$)', re.IGNORECASE), re.compile(r'\.(pem|key|p12|pfx|cert|crt|der|p8)$', re.IGNORECASE), - re.compile(r'(id_rsa|id_dsa|id_ecdsa|id_ed25519)(\.pub)?$'), - re.compile(r'(\.netrc|\.pgpass|\.htpasswd)$', re.IGNORECASE), - re.compile(r'(aws_credentials|gcloud_credentials|service.account)', re.IGNORECASE), + # SSH/GPG private keys. Left boundary + IGNORECASE so `grid_rsa` (alpha before + # `id_rsa`) and `ID_RSA` are handled correctly, not matched as a substring. + re.compile(r'(^|[^A-Za-z0-9])(id_rsa|id_dsa|id_ecdsa|id_ed25519)(\.pub)?$', re.IGNORECASE), + re.compile(r'^secring(\.(gpg|pgp))?$', re.IGNORECASE), # GPG private keyring + # Auth/credential dotfiles that routinely hold tokens (#2106: .npmrc/.pypirc/ + # .git-credentials/.boto were silently indexed before). + re.compile(r'(\.netrc|\.pgpass|\.htpasswd|\.npmrc|\.pypirc|\.git-credentials|\.boto)$', re.IGNORECASE), + # NOTE: aws_credentials/gcloud_credentials/service_account moved to the + # boundary-checked Stage 3 keyword path (#2106). The old unbounded + # `service.account` substring (regex `.` wildcard) matched real source like + # google/oauth2/service_account.py and prose like aws_credentials_rotation.md. ] # Generic keyword patterns - these only count when the keyword is LOAD-BEARING @@ -123,12 +142,51 @@ def _zip_within_caps(path: Path) -> bool: _GENERIC_KEYWORD_PATTERNS = [ re.compile(r'(? bool: + """A prose/note file (.md/.rst/...) whose stem is a multi-word topic slug is + exempt from the generic-keyword drop (#2106). A stem that IS exactly a bare + keyword (secrets / token / passwords) is NOT exempt — that still reads as a + credential dump.""" + if path.suffix.lower() not in _PROSE_EXTS: + return False + stem = Path(path.name).stem.lstrip('.') or Path(path.name).stem + return not any(p.fullmatch(stem) for p in _GENERIC_KEYWORD_PATTERNS) def _generic_keyword_hit(name: str) -> bool: @@ -141,9 +199,11 @@ def _generic_keyword_hit(name: str) -> bool: ("token-economics-of-recall.md", "password-policy-discussion.md") and must not cause the file to be silently dropped from the graph (#436, #718). """ - # Stem = name up to the first dot, ignoring leading dots so dotfiles like - # ".token" keep their keyword ("" stems would never match). - stem = name.lstrip('.').split('.')[0] + # Stem = name minus only the FINAL extension (not up to the first dot), so a + # multi-dot topic slug like `token.economics.notes.md` keeps all its words and + # doesn't collapse to a bare `token` (#2106). Leading dots stripped so + # dotfiles like `.token` keep their keyword. + stem = Path(name).stem.lstrip('.') or Path(name).stem for pat in _GENERIC_KEYWORD_PATTERNS: hit = False for m in pat.finditer(stem): @@ -173,19 +233,48 @@ def _generic_keyword_hit(name: str) -> bool: _PAPER_SIGNAL_THRESHOLD = 3 # need at least this many signals to call it a paper +def _is_graphable_source(path: Path) -> bool: + """True for genuine programming-language source — the only category exempt + from the ambiguous-dir (Stage 1, #1943) and generic-keyword (Stage 3, #1666) + drops. Data/serialization formats are NOT exempt even though some route + through the CODE path for manifest parsing: credentials.json / secrets.yaml + are exactly the stores those stages must keep catching. + """ + return classify_file(path) == FileType.CODE and path.suffix.lower() not in _SECRET_PRONE_DATA_EXTS + + def _is_sensitive(path: Path) -> bool: """Return True if this file likely contains secrets and should be skipped.""" # Stage 1: any PARENT directory is a known secrets dir (parts[:-1] excludes # the filename itself so a root-level file named "credentials" is not falsely - # skipped — the name patterns in Stage 2 handle the filename). - if any(part in _SENSITIVE_DIRS for part in path.parts[:-1]): + # skipped — the name patterns in Stage 2 handle the filename). Dedicated + # credential stores drop everything unconditionally; ambiguous bare-name dirs + # (secrets/, credentials/) spare genuine source (#1943), which still falls + # through so Stages 2-3 screen its filename like anywhere else. + parents = path.parts[:-1] + # Lowercase the segment comparison so `Secrets/`/`SECRETS/` (real on + # case-insensitive macOS/Windows filesystems) are still caught (#2106). + if any(part.lower() in _CREDENTIAL_STORE_DIRS for part in parents): + return True + if any(part.lower() in _AMBIGUOUS_SENSITIVE_DIRS for part in parents) and not _is_graphable_source(path): return True # Stage 2: filename pattern match name = path.name if any(p.search(name) for p in _SENSITIVE_PATTERNS): return True - # Stage 3: generic keywords, only when load-bearing in the name - return _generic_keyword_hit(name) + # Stage 3: generic keywords, only when load-bearing in the name. Do NOT let a + # bare name keyword silently drop a genuine programming-language source file: + # a .rb/.py named device_token or passwords_controller is a module, not a secret + # store (#1666). Data/config formats (.json, .yaml, .toml, ...) are deliberately + # NOT exempt even though .json routes through the CODE path for manifest parsing, + # because credentials.json / oauth_token.json / secrets.yaml are exactly the + # secret stores this stage must catch. The specific Stage 2 patterns (.env, .pem, + # id_rsa, ...) still apply to everything regardless of extension. + if _generic_keyword_hit(name): + # Genuine source AND multi-word prose notes are exempt; a bare-keyword + # name (secrets.md, token.txt) still drops (#1666, #2106). + return not (_is_graphable_source(path) or _is_prose_note(path)) + return False def _looks_like_paper(path: Path) -> bool: @@ -601,7 +690,7 @@ def _edge(src: str, tgt: str, relation: str) -> None: return {"nodes": nodes, "edges": edges} -def convert_office_file(path: Path, out_dir: Path) -> Path | None: +def convert_office_file(path: Path, out_dir: Path, root: "Path | None" = None) -> Path | None: """Convert a .docx or .xlsx to a markdown sidecar in out_dir. Returns the path of the converted .md file, or None if conversion failed @@ -620,14 +709,30 @@ def convert_office_file(path: Path, out_dir: Path) -> Path | None: out_dir.mkdir(parents=True, exist_ok=True) # Use a stable name derived from the original path to avoid collisions. - # Normalize the resolved path to NFC before hashing: on macOS (HFS+/APFS) - # os.walk/rglob return filenames in NFD, while Python string literals and - # directly-constructed Path objects are NFC, so the same source file would - # otherwise hash to different sidecar names across runs — causing --update - # to treat every Office file as new and re-extract it (#1226). + # Hash the path RELATIVE to the scan root, not the absolute path: the + # absolute form salts the name with the checkout location, so the same + # tracked .xlsx in two clones/worktrees emits two differently-named, + # byte-identical sidecars — unbounded duplicates when graphify-out/ is + # committed, each ingested as a distinct source doc (#2059). The relative + # path still disambiguates same-stem files in different directories. + # Normalize to NFC before hashing: on macOS (HFS+/APFS) os.walk/rglob return + # filenames in NFD, while Python string literals and directly-constructed + # Path objects are NFC, so the same source file would otherwise hash to + # different sidecar names across runs — making --update treat every Office + # file as new and re-extract it (#1226). import hashlib import unicodedata - normalized_path = unicodedata.normalize("NFC", str(path.resolve())) + if root is None: + # Default layout: out_dir is //converted. + root = out_dir.parent.parent + try: + key = path.resolve().relative_to(Path(root).resolve()).as_posix() + except (ValueError, OSError): + # Not under the scan root (custom GRAPHIFY_OUT layouts, --include + # sources, direct API callers): keep the previous absolute form rather + # than guessing, so behavior is unchanged for those cases. + key = str(path.resolve()) + normalized_path = unicodedata.normalize("NFC", key) name_hash = hashlib.sha256(normalized_path.encode()).hexdigest()[:8] out_path = out_dir / f"{path.stem}_{name_hash}.md" # Skip re-writing only when the sidecar is present AND at least as new as the @@ -667,17 +772,17 @@ def count_words(path: Path) -> int: # Directory names to always skip - venvs, caches, build artifacts, deps _SKIP_DIRS = { - "venv", ".venv", "env", ".env", + "venv", ".venv", # "env"/".env"/"*_env" are gated on venv markers below (#2058) "node_modules", "__pycache__", ".git", "dist", "build", "target", "out", "site-packages", "lib64", ".pytest_cache", ".mypy_cache", ".ruff_cache", - ".tox", ".eggs", "*.egg-info", + ".tox", ".nox", ".eggs", "*.egg-info", # nox is tox's successor, same .nox/ venv shape (#1804) "graphify-out", GRAPHIFY_OUT_NAME, # never treat own output as source input (#524); honour GRAPHIFY_OUT (#1423) # Coverage/test-artefact dirs — generated, never architecturally meaningful "coverage", "lcov-report", # Vitest/Istanbul/nyc HTML reports (#870) "visual-tests", "visual-test", # Playwright/visual-regression bundles (#869) - "__snapshots__", "snapshots", # Jest/Vitest snapshot dirs + "__snapshots__", # Jest/Vitest snapshot dir (unambiguous) "storybook-static", # Storybook production build output "dist-protected", # Protected dist variants (same noise as dist) # Framework cache/build dirs — generated, never architecturally meaningful (#873) @@ -692,14 +797,68 @@ def count_words(path: Path) -> int: "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "Cargo.lock", "poetry.lock", "Gemfile.lock", "composer.lock", "go.sum", "go.work.sum", + # Removed allowlist config (#2112) — no longer consumed, so keep a leftover + # file out of the unclassified list instead of surfacing it as scan input. + ".graphifyinclude", } +# A bare "snapshots" dir is a Jest/Vitest artifact only when it actually holds +# snapshot files or lives directly under a JS test root. Elsewhere it is often a +# real code namespace (e.g. Rails app/services/snapshots/), so pruning it by name +# silently dropped legitimate source from the graph (#1666). "__snapshots__" stays +# unconditionally pruned above; only the ambiguous bare name is gated here. +_JS_SNAPSHOT_TEST_ROOTS = frozenset({"__tests__", "__test__"}) + + +def _has_venv_markers(d: "Path") -> bool: + """True only when *d* has actual virtualenv/conda structure on disk. + + ``env``/``.env``/``*_env`` is a real source-directory convention (UVM/ASIC + verification trees, and others), so pruning it by name alone silently drops + legitimate source with no trace (#2058). Prune it only on real evidence: a + ``pyvenv.cfg``, an ``activate`` script, a ``lib/python*`` tree, or conda's + ``conda-meta/`` (``conda create -p ./env`` writes no pyvenv.cfg). + """ + try: + if (d / "pyvenv.cfg").is_file(): + return True + if (d / "bin" / "activate").is_file() or (d / "Scripts" / "activate").is_file(): + return True + if next(d.glob("lib/python*"), None) is not None: + return True + if (d / "conda-meta").is_dir(): + return True + except OSError: + pass + return False + + def _is_noise_dir(part: str, parent: "Path | None" = None) -> bool: """Return True if this directory name looks like a venv, cache, or dep dir.""" if part in _SKIP_DIRS: return True - # Catch *_venv, *_repo/site-packages patterns - if part.endswith("_venv") or part.endswith("_env"): + if part in ("env", ".env") or part.endswith("_env"): + # Ambiguous: a real venv OR a real source dir. Prune only on actual venv + # evidence, mirroring the "snapshots" gating (#1666/#2058). + if parent is None: + return False # cannot verify; keep a possibly-real code dir + return _has_venv_markers(parent / part) + if part == "snapshots": + # Prune only when it looks like an actual JS/Vitest snapshot dir. + if parent is None: + return False # cannot verify; keep a possibly-real code dir + snap_dir = parent / part + if parent.name in _JS_SNAPSHOT_TEST_ROOTS: + return True + try: + if next(snap_dir.glob("*.snap"), None) is not None: + return True + except OSError: + pass + return False + # Catch *_venv (unambiguous — "venv" is always a virtualenv signal). "*_env" + # is gated on markers above (#2058), not pruned by name. + if part.endswith("_venv"): return True if part.endswith(".egg-info"): return True @@ -749,7 +908,76 @@ def _find_vcs_root(start: Path) -> Path | None: current = parent -def _load_graphifyignore(root: Path) -> list[tuple[Path, str]]: +def _git_info_exclude(vcs_root: Path) -> Path | None: + """Resolve ``$GIT_DIR/info/exclude`` for the repo rooted at ``vcs_root``. + + ``info/exclude`` is where git records local-only, uncommitted excludes — and + where ``git worktree add`` writes nested worktree paths — so a repo can ignore + a directory without any ``.gitignore`` entry. graphify only read + ``.gitignore``/``.graphifyignore``, so it walked into those worktree copies and + the graph exploded (#1810). Handles the linked-worktree/submodule case where + ``.git`` is a file (``gitdir: ``) and the real excludes live in the + shared common git dir. Returns None when there is no readable exclude file. + """ + dot_git = vcs_root / ".git" + git_dir: Path | None = None + if dot_git.is_dir(): + git_dir = dot_git + elif dot_git.is_file(): + try: + content = dot_git.read_text(encoding="utf-8", errors="ignore").strip() + except OSError: + content = "" + if content.startswith("gitdir:"): + gd = Path(content[len("gitdir:"):].strip()) + if not gd.is_absolute(): + gd = (vcs_root / gd).resolve() + git_dir = gd + # A linked worktree's gitdir holds a `commondir` file pointing at the + # shared git dir, where info/exclude actually lives. + commondir = gd / "commondir" + if commondir.exists(): + try: + cd_raw = commondir.read_text(encoding="utf-8", errors="ignore").strip() + except OSError: + cd_raw = "" + if cd_raw: + cd = Path(cd_raw) + git_dir = cd if cd.is_absolute() else (gd / cd).resolve() + if git_dir is None: + return None + exclude = git_dir / "info" / "exclude" + return exclude if exclude.is_file() else None + + +def _load_dir_own_ignore(d: Path, *, gitignore: bool = True) -> list[tuple[Path, str]]: + """Read .gitignore/.graphifyignore directly inside *d* (not its ancestors). + + Merges .gitignore and .graphifyignore for this one directory (#1363): + .gitignore is read first and .graphifyignore last, so .graphifyignore + patterns (including `!` negations) win on conflict via last-match-wins; + adding a .graphifyignore can only ever exclude MORE, never re-include a + .gitignore-excluded file (#945 kept: a dir with only a .gitignore still + gets sensible defaults). + + Shared by `_load_graphifyignore` (ancestor chain, loaded once before the + scan) and the live os.walk loop in `detect()` (called per-directory as + each descendant is visited), so nested ignore files *below* the scan + root are honored too — previously only the scan root and its ancestors + were read, so e.g. `vendor/sub/.gitignore` was silently ignored (#1206). + """ + patterns: list[tuple[Path, str]] = [] + for fname in ((".gitignore", ".graphifyignore") if gitignore else (".graphifyignore",)): + ignore_file = d / fname + if ignore_file.exists(): + for raw in ignore_file.read_text(encoding="utf-8-sig", errors="ignore").splitlines(): + line = _parse_gitignore_line(raw) + if line: + patterns.append((d, line)) + return patterns + + +def _load_graphifyignore(root: Path, *, gitignore: bool = True) -> list[tuple[Path, str]]: """Read .graphifyignore files and return (anchor_dir, pattern) pairs. Patterns are returned outer-first so that inner (closer) rules are @@ -758,6 +986,10 @@ def _load_graphifyignore(root: Path) -> list[tuple[Path, str]]: Walk ceiling: the nearest VCS root if inside a repo, otherwise the scan root itself (hermetic — no leakage across unrelated sibling projects). + + Covers the scan root and its ancestors only — directories *below* the + scan root are picked up live during the os.walk in `detect()` instead, + since they aren't known until the walk reaches them (#1206). """ root = root.resolve() ceiling = _find_vcs_root(root) or root @@ -773,27 +1005,51 @@ def _load_graphifyignore(root: Path) -> list[tuple[Path, str]]: dirs.reverse() # ceiling first, scan root last patterns: list[tuple[Path, str]] = [] + + # $GIT_DIR/info/exclude is repo-root-scoped and, per git, ranks below every + # per-directory .gitignore/.graphifyignore — so load it first (lowest priority + # under last-match-wins) anchored at the VCS root, letting a nearer `!` + # re-include still override it (#1810). + info_exclude = _git_info_exclude(ceiling) if gitignore else None + if info_exclude is not None: + for raw in info_exclude.read_text(encoding="utf-8-sig", errors="ignore").splitlines(): + line = _parse_gitignore_line(raw) + if line: + patterns.append((ceiling, line)) + for d in dirs: - # Merge .gitignore and .graphifyignore for this dir (#1363). Previously - # the presence of a .graphifyignore made graphify skip that dir's - # .gitignore entirely, so a file excluded only by .gitignore (e.g. a - # neutrally-named secret like prod-dump.sql) silently got indexed into - # the graph — whose artifacts embed file contents and are often - # committed. .gitignore is read first and .graphifyignore last, so - # .graphifyignore patterns (including `!` negations) win on conflict via - # last-match-wins; adding a .graphifyignore can only ever exclude MORE, - # never re-include a .gitignore-excluded file (#945 kept: a project with - # only a .gitignore still gets sensible defaults). - for fname in (".gitignore", ".graphifyignore"): - ignore_file = d / fname - if ignore_file.exists(): - for raw in ignore_file.read_text(encoding="utf-8", errors="ignore").splitlines(): - line = _parse_gitignore_line(raw) - if line: - patterns.append((d, line)) + patterns.extend(_load_dir_own_ignore(d, gitignore=gitignore)) return patterns +def _match_anchored_ignore_pattern(path: str, pattern: str) -> bool: + """Match an anchored gitignore pattern without letting ``*`` cross ``/``.""" + path_parts = tuple(path.split("/")) + pattern_parts = tuple(pattern.split("/")) + + @lru_cache(maxsize=None) + def _matches(path_idx: int, pattern_idx: int) -> bool: + if pattern_idx == len(pattern_parts): + return path_idx == len(path_parts) + + part = pattern_parts[pattern_idx] + if part == "**": + if pattern_idx == len(pattern_parts) - 1: + return path_idx < len(path_parts) + return _matches(path_idx, pattern_idx + 1) or ( + path_idx < len(path_parts) + and _matches(path_idx + 1, pattern_idx) + ) + + return ( + path_idx < len(path_parts) + and fnmatch.fnmatchcase(path_parts[path_idx], part) + and _matches(path_idx + 1, pattern_idx + 1) + ) + + return _matches(0, 0) + + def _is_ignored( path: Path, root: Path, @@ -821,9 +1077,9 @@ def _eval(target: Path) -> bool: """Apply last-match-wins to a single target path.""" if _cache is not None and target in _cache: return _cache[target] - def _matches(rel: str, p: str, anchored: bool) -> bool: - if anchored: - return fnmatch.fnmatch(rel, p) + def _matches(rel: str, p: str, path_relative: bool) -> bool: + if path_relative: + return _match_anchored_ignore_pattern(rel, p) parts = rel.split("/") if fnmatch.fnmatch(rel, p): return True @@ -840,30 +1096,26 @@ def _matches(rel: str, p: str, anchored: bool) -> bool: for anchor, pattern in patterns: negated = pattern.startswith("!") raw = pattern[1:] if negated else pattern - anchored = raw.startswith("/") + directory_only = raw.endswith("/") + path_relative = "/" in raw.rstrip("/") p = raw.strip("/") if not p: continue + # gitignore semantics: patterns from A/.gitignore apply ONLY to paths + # under A. Matching non-anchored patterns against root-relative paths + # let e.g. .hypothesis/.gitignore's bare "*" ignore the ENTIRE repo + # (detect() returned 0 files). The anchor dir itself is exempt — an + # ignore file governs its directory's contents, not the directory. matched = False - if anchored: - try: - rel_anchor = str(target.relative_to(anchor)).replace(os.sep, "/") - matched = _matches(rel_anchor, p, anchored=True) - except ValueError: - pass - else: - try: - rel = str(target.relative_to(root)).replace(os.sep, "/") - matched = _matches(rel, p, anchored=False) - except ValueError: - pass - if not matched and anchor != root: - try: - rel_anchor = str(target.relative_to(anchor)).replace(os.sep, "/") - matched = _matches(rel_anchor, p, anchored=False) - except ValueError: - pass + try: + rel_anchor = str(target.relative_to(anchor)).replace(os.sep, "/") + except ValueError: + continue # target outside this pattern's anchor: cannot match + if rel_anchor != ".": + matched = _matches(rel_anchor, p, path_relative=path_relative) + if matched and directory_only and not target.is_dir(): + matched = False if matched: result = not negated # last match wins; ! flips to un-ignore @@ -888,117 +1140,6 @@ def _matches(rel: str, p: str, anchored: bool) -> bool: return _eval(path) -def _load_graphifyinclude(root: Path) -> list[tuple[Path, str]]: - """Read .graphifyinclude allowlist patterns from root and ancestors. - - Include patterns opt matching hidden files/dirs into traversal. Sensitive - files and hard-skipped noise directories are still excluded later. - Uses the same VCS-root ceiling logic as _load_graphifyignore. - """ - root = root.resolve() - ceiling = _find_vcs_root(root) or root - - dirs: list[Path] = [] - current = root - while True: - dirs.append(current) - if current == ceiling: - break - current = current.parent - dirs.reverse() - - patterns: list[tuple[Path, str]] = [] - for d in dirs: - include_file = d / ".graphifyinclude" - if include_file.exists(): - for raw in include_file.read_text(encoding="utf-8", errors="ignore").splitlines(): - line = _parse_gitignore_line(raw) - if line: - patterns.append((d, line)) - return patterns - - -def _is_included(path: Path, root: Path, patterns: list[tuple[Path, str]]) -> bool: - """Return True if path matches any .graphifyinclude allowlist pattern.""" - if not patterns: - return False - - def _matches(rel: str, p: str, anchored: bool) -> bool: - if anchored: - return fnmatch.fnmatch(rel, p) - parts = rel.split("/") - if fnmatch.fnmatch(rel, p): - return True - if fnmatch.fnmatch(path.name, p): - return True - for i, part in enumerate(parts): - if fnmatch.fnmatch(part, p): - return True - if fnmatch.fnmatch("/".join(parts[:i + 1]), p): - return True - return False - - for anchor, pattern in patterns: - anchored = pattern.startswith("/") - p = pattern.strip("/") - if not p: - continue - if anchored: - try: - rel_anchor = str(path.relative_to(anchor)).replace(os.sep, "/") - if _matches(rel_anchor, p, anchored=True): - return True - except ValueError: - pass - else: - try: - rel = str(path.relative_to(root)).replace(os.sep, "/") - if _matches(rel, p, anchored=False): - return True - except ValueError: - pass - if anchor != root: - try: - rel_anchor = str(path.relative_to(anchor)).replace(os.sep, "/") - if _matches(rel_anchor, p, anchored=False): - return True - except ValueError: - pass - return False - - -def _could_contain_included_path(path: Path, root: Path, patterns: list[tuple[Path, str]]) -> bool: - """Return True if a directory may contain files matched by .graphifyinclude.""" - if not patterns: - return False - - rels: list[str] = [] - try: - rels.append(str(path.relative_to(root)).replace(os.sep, "/")) - except ValueError: - pass - for anchor, _ in patterns: - if anchor != root: - try: - rels.append(str(path.relative_to(anchor)).replace(os.sep, "/")) - except ValueError: - pass - - for rel in rels: - rel = rel.strip("/") - if not rel: - return True - for _, pattern in patterns: - p = pattern.strip("/") - if not p: - continue - if p == rel or p.startswith(rel + "/"): - return True - if fnmatch.fnmatch(rel, p): - return True - return False - - def _auto_follow_symlinks(root: Path) -> bool: """Return whether ``root`` has any direct symlinked child. @@ -1024,8 +1165,21 @@ def _resolves_under_root(path: Path, root: Path) -> bool: return True -def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: bool | None = None, extra_excludes: list[str] | None = None) -> dict: +def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: bool | None = None, extra_excludes: list[str] | None = None, cache_root: Path | None = None, gitignore: bool = True) -> dict: root = root.resolve() + # .graphifyinclude support was removed (#2112): its loader and matchers had + # no consumers, so the file has been a silent no-op since dot directories + # became indexed by default (#873). Surface that once per scan so a + # leftover allowlist file is not a silent behavior change. + if (root / ".graphifyinclude").is_file(): + import sys as _sys + print( + "[graphify] WARNING: .graphifyinclude is no longer supported " + "(it has been non-functional since dot directories became indexed " + "by default); to re-include ignored paths, use ! negation patterns " + "in .graphifyignore.", + file=_sys.stderr, + ) if follow_symlinks is None: follow_symlinks = False google_workspace = google_workspace_enabled() if google_workspace is None else google_workspace @@ -1041,11 +1195,20 @@ def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: def _wc(path: Path) -> int: # Cache word counts against each file's stat signature so unchanged # PDFs/docx aren't re-parsed on every run just to size the corpus (#1656). + # cache_root (when given, e.g. from `extract --out`) keeps this cache out + # of the scanned corpus (#1747). from graphify import cache as _cache - return _cache.cached_word_count(path, root, count_words) + return _cache.cached_word_count(path, root, count_words, cache_root=cache_root) skipped_sensitive: list[str] = [] - ignore_patterns = _load_graphifyignore(root) + unclassified: list[str] = [] + # Files/dirs dropped by a .gitignore/.graphifyignore rule. Recorded so an + # over-broad ignore (or a legitimately-ignored subtree) is visible instead + # of silently vanishing from the graph (#1922). Directory-level entries keep + # this bounded — a pruned `data/` is one entry, not one per contained file. + ignored: list[str] = [] + pruned_noise: list[str] = [] + ignore_patterns = _load_graphifyignore(root, gitignore=gitignore) ignore_cache: dict[Path, bool] = {} # shared across all _is_ignored calls in this scan # CLI --exclude patterns are anchored at the scan root and appended last # so they win over any .graphifyignore/.gitignore rules (#947). @@ -1054,7 +1217,6 @@ def _wc(path: Path) -> int: line = _parse_gitignore_line(pat) if line: ignore_patterns.append((root, line)) - include_patterns = _load_graphifyinclude(root) # Always include graphify-out/memory/ - query results filed back into the graph memory_dir = root / GRAPHIFY_OUT / "memory" @@ -1065,9 +1227,29 @@ def _wc(path: Path) -> int: seen: set[Path] = set() all_files: list[Path] = [] + # os.walk swallows os.scandir errors by default (no onerror -> the failing + # directory subtree is silently skipped). That turns a transient + # PermissionError, or a directory created/deleted mid-walk (e.g. concurrent + # writes racing the scan), into a partial file list and, downstream, a + # silently partial graph.json. Record and surface every skipped directory + # so an incomplete enumeration is visible rather than silent. + walk_errors: list[str] = [] + + def _on_walk_error(err: OSError) -> None: + import sys as _sys + target = getattr(err, "filename", None) or "" + walk_errors.append(f"{target}: {err}") + print( + f"[graphify] WARNING: could not scan {target} ({err}); " + f"its files are missing from this run's enumeration.", + file=_sys.stderr, + ) + for scan_root in scan_paths: in_memory_tree = memory_dir.exists() and str(scan_root).startswith(str(memory_dir)) - for dirpath, dirnames, filenames in os.walk(scan_root, followlinks=follow_symlinks): + for dirpath, dirnames, filenames in os.walk( + scan_root, followlinks=follow_symlinks, onerror=_on_walk_error + ): dp = Path(dirpath) if follow_symlinks and os.path.islink(dirpath): real = os.path.realpath(dirpath) @@ -1076,6 +1258,14 @@ def _wc(path: Path) -> int: dirnames.clear() continue if not in_memory_tree: + # dp == root was already loaded by _load_graphifyignore (root is + # the last entry in its ancestor chain); every other directory + # reached by the walk is a descendant below the scan root, whose + # own .gitignore/.graphifyignore is unknown until we get here. + # Load it now, before pruning dp's children, so a nested ignore + # file governs its own subtree the same way git honors it (#1206). + if dp != root: + ignore_patterns.extend(_load_dir_own_ignore(dp, gitignore=gitignore)) # Prune noise dirs in-place so os.walk never descends into them. # Dot dirs are allowed — users often want .github/, .claude/, etc. # Framework caches (.next, .nuxt, …) are caught by _is_noise_dir. @@ -1088,11 +1278,19 @@ def _wc(path: Path) -> int: # any `!` rule existed — e.g. a single `!docs/**` made the walk descend # bin/, obj/, wwwroot/, generated/, … : a pathological slowdown on large # repos for no correctness gain. - dirnames[:] = [ - d for d in dirnames - if not _is_noise_dir(d, dp) - and not _is_ignored(dp / d, root, ignore_patterns, _cache=ignore_cache) - ] + kept_dirs: list[str] = [] + for d in dirnames: + if _is_noise_dir(d, dp): + # Record pruned-as-noise dirs so a wrongly-pruned real + # source dir is at least traceable in the output rather + # than vanishing silently (#2058). + pruned_noise.append(str(dp / d) + os.sep) + continue + if _is_ignored(dp / d, root, ignore_patterns, _cache=ignore_cache): + ignored.append(str(dp / d) + os.sep) + continue + kept_dirs.append(d) + dirnames[:] = kept_dirs if follow_symlinks: safe_dirs: list[str] = [] for d in dirnames: @@ -1122,6 +1320,7 @@ def _wc(path: Path) -> int: if str(p).startswith(str(converted_dir)): continue if not in_memory and _is_ignored(p, root, ignore_patterns, _cache=ignore_cache): + ignored.append(str(p)) continue if not _resolves_under_root(p, root): skipped_sensitive.append(str(p) + " [symlink target outside scan root]") @@ -1130,6 +1329,13 @@ def _wc(path: Path) -> int: skipped_sensitive.append(str(p)) continue ftype = classify_file(p) + if not ftype: + # Considered but unclassifiable: an extension not in any supported set, + # or an extensionless, non-shebang file (Dockerfile, Gemfile, Makefile, + # Rakefile, LICENSE, ...). Previously these left no trace at all — not + # counted, not listed — so a user couldn't tell they were seen (#1692). + unclassified.append(str(p)) + continue if ftype: if p.suffix.lower() in GOOGLE_WORKSPACE_EXTENSIONS: if not google_workspace: @@ -1140,7 +1346,7 @@ def _wc(path: Path) -> int: ) continue try: - md_path = convert_google_workspace_file(p, converted_dir, xlsx_to_markdown=xlsx_to_markdown) + md_path = convert_google_workspace_file(p, converted_dir, xlsx_to_markdown=xlsx_to_markdown, root=root) except Exception as exc: skipped_sensitive.append(str(p) + f" [Google Workspace export failed: {exc}]") continue @@ -1154,7 +1360,7 @@ def _wc(path: Path) -> int: continue # Office files: convert to markdown sidecar so subagents can read them if p.suffix.lower() in OFFICE_EXTENSIONS: - md_path = convert_office_file(p, converted_dir) + md_path = convert_office_file(p, converted_dir, root=root) if md_path: if _is_ignored(md_path, root, ignore_patterns, _cache=ignore_cache): continue @@ -1195,6 +1401,10 @@ def _wc(path: Path) -> int: "needs_graph": needs_graph, "warning": warning, "skipped_sensitive": skipped_sensitive, + "unclassified": sorted(unclassified), + "walk_errors": walk_errors, + "ignored": sorted(ignored), + "pruned_noise_dirs": sorted(pruned_noise), "graphifyignore_patterns": len(ignore_patterns), "scan_root": str(root.resolve()), } @@ -1250,6 +1460,18 @@ def _stat_and_hash(path_str: str) -> tuple[str, float, str] | None: return None +def _nfc(s: str) -> str: + """NFC-normalize a path string used as a manifest key. + + On macOS, ``os.walk`` / ``getcwd`` yield NFD paths while path literals + and many skill-substituted roots are NFC. Raw string compare then treats + every file as both deleted and new, forcing a full re-extract (#2221). + Same boundary as the Office sidecar hash fix (#1226). + """ + import unicodedata + return unicodedata.normalize("NFC", s) + + def _to_relative_for_storage(key: str, root: Path) -> str: """Return ``key`` as a forward-slash relative path from ``root``. @@ -1264,12 +1486,18 @@ def _to_relative_for_storage(key: str, root: Path) -> str: under its own name. Resolving the key would point the stored entry at the symlink target, and the original key would then miss on reload and re-extract on every incremental run. + + Both sides of ``relpath`` are NFC'd first: stamped keys may already be + NFC while ``Path(root).resolve()`` is NFD on macOS, and a mixed-form + compare would mark an in-root file as ``../…`` and keep it absolute + (#2221 / #777). """ p = Path(key) if not p.is_absolute(): return key try: - rel = os.path.relpath(p, Path(root).resolve()) + base = _nfc(str(Path(root).resolve())) + rel = os.path.relpath(_nfc(str(p)), base) except (ValueError, OSError): return key # outside root (e.g. Windows cross-drive) # ``os.path.relpath`` happily produces ``../foo`` for paths outside @@ -1288,11 +1516,15 @@ def _to_absolute_from_storage(key: str, root: Path) -> str: that newly-loaded manifests from before this change remain readable. Uses ``Path(root).resolve()`` so the produced absolute path matches what :func:`detect` returns (which also resolves the scan root). + NFC both sides so a relative key and an NFD-resolved root still join + to the same string form the rest of the manifest path uses (#2221). """ p = Path(key) if p.is_absolute(): return str(p) - return str(Path(root).resolve() / p) + # NFC the joined result so an NFD-resolved root + relative key lands on + # the same form load_manifest / detect_incremental compare against. + return _nfc(str(Path(root).resolve() / p)) def load_manifest( @@ -1307,14 +1539,19 @@ def load_manifest( manifests with absolute keys pass through unchanged, so a graphify-out/ written by an older version (or by a caller that didn't supply ``root`` to :func:`save_manifest`) remains readable. + + Keys are NFC-normalized on load so a manifest written under one Unicode + form still matches a scan that yields the other (#2221). """ try: raw = json.loads(Path(manifest_path).read_text(encoding="utf-8")) except Exception: return {} - if root is None or not isinstance(raw, dict): + if not isinstance(raw, dict): return raw - return {_to_absolute_from_storage(k, root): v for k, v in raw.items()} + if root is None: + return {_nfc(k): v for k, v in raw.items()} + return {_nfc(_to_absolute_from_storage(k, root)): v for k, v in raw.items()} def save_manifest( @@ -1323,6 +1560,8 @@ def save_manifest( *, kind: str = "both", root: Path | None = None, + scan_corpus: set[str] | list[str] | None = None, + clear_semantic: set[str] | list[str] | None = None, ) -> None: """Save current file mtimes + content hashes for change detection. @@ -1338,9 +1577,81 @@ def save_manifest( machines and checkout locations (#777). Out-of-root entries are written as absolute so they continue to round-trip on the saving machine. When ``root`` is None the legacy absolute-keyed format is preserved. + + ``scan_corpus`` (#1908): full-scan callers pass the COMPLETE detect + corpus (absolute paths) so seeded rows for in-root files that are still + alive on disk but no longer part of the scan (newly excluded via + .graphifyignore/.gitignore/--exclude) are dropped instead of surviving + forever and masquerading as deletions in detect_incremental. It must be + the RAW detect output, not a stamp-filtered subset — pruning to a + filtered set would erase rows the filter merely omitted (failed chunks, + --code-only doc rows). Out-of-root entries are never pruned. Callers + saving a SUBSET of files (changed_paths hooks, skill runbooks, #917) + must leave this None so their untouched rows are preserved. + + ``clear_semantic`` (#1948): files that were dispatched this run but + produced no stamped output (e.g. the LLM omitted their chunk on a + --force re-run) are absent from ``files``, so the seed loop below would + otherwise copy their prior semantic_hash verbatim — masking the omission + and making detect_incremental(kind="semantic") report them unchanged. + Pass the set of such files (any path form ``scan_corpus`` accepts) to + force their seeded semantic_hash to "" instead of inheriting it. """ existing = load_manifest(manifest_path, root=root) + # Index both raw and NFC forms so scan/clear membership survives the + # same NFC/NFD mismatch that breaks manifest lookups (#2221). + def _path_index(paths: set[str] | list[str] | None) -> set[str] | None: + if paths is None: + return None + indexed: set[str] = set() + for p in paths: + indexed.add(p) + indexed.add(_nfc(p)) + return indexed + + scan_set = _path_index(scan_corpus) + clear_set = _path_index(clear_semantic) + try: + root_res: Path | None = Path(root).resolve() if root is not None else None + except (OSError, RuntimeError): + root_res = Path(root) if root is not None else None + + def _in_scan(path_str: str) -> bool: + if path_str in scan_set or _nfc(path_str) in scan_set: + return True + try: + resolved = str(Path(path_str).resolve()) + return resolved in scan_set or _nfc(resolved) in scan_set + except (OSError, RuntimeError): + return False + + def _in_clear(path_str: str) -> bool: + if path_str in clear_set or _nfc(path_str) in clear_set: + return True + try: + resolved = str(Path(path_str).resolve()) + return resolved in clear_set or _nfc(resolved) in clear_set + except (OSError, RuntimeError): + return False + + def _in_root(path_str: str) -> bool: + # Without a root we cannot tell in-root from out-of-root; fail open + # (keep the row) so out-of-root corpora are never pruned by accident. + if root_res is None: + return False + p = Path(path_str) + try: + p.relative_to(root_res) + return True + except ValueError: + pass + try: + p.resolve().relative_to(root_res) + return True + except (ValueError, OSError, RuntimeError): + return False + def _normalise_entry(entry): if isinstance(entry, (int, float)): return {"mtime": entry, "ast_hash": "", "semantic_hash": ""} @@ -1353,17 +1664,27 @@ def _normalise_entry(entry): # Seed from the existing manifest so incremental callers passing a subset # of files don't silently erase entries for untouched files (#917). # Prune entries whose file no longer exists on disk — those are genuine - # deletions that detect_incremental() should treat as gone. + # deletions that detect_incremental() should treat as gone. When the + # caller supplied the full scan corpus, additionally prune in-root rows + # the scan no longer covers: those files were excluded, not deleted, and + # keeping the row makes them look deleted on every future run (#1908). manifest: dict[str, dict] = {} for f, entry in existing.items(): normalised = _normalise_entry(entry) if normalised is None: continue try: - if Path(f).exists(): - manifest[f] = normalised + if not Path(f).exists(): + continue except OSError: continue + if scan_set is not None and not _in_scan(f) and _in_root(f): + continue # excluded-but-alive: drop the stale row (#1908) + if clear_set is not None and _in_clear(f): + # Dispatched-but-omitted this run: don't inherit the stale + # semantic_hash, or detect_incremental would call it unchanged (#1948). + normalised = {**normalised, "semantic_hash": ""} + manifest[f] = normalised all_files = [f for file_list in files.values() for f in file_list] with ThreadPoolExecutor() as pool: @@ -1376,7 +1697,8 @@ def _normalise_entry(entry): if f not in hashed: continue # file deleted between detect() and manifest write mtime, h = hashed[f] - prev = _normalise_entry(existing.get(f, {})) or {} + key = _nfc(f) + prev = _normalise_entry(existing.get(key, {})) or {} entry: dict = {"mtime": mtime} if kind in ("ast", "both"): entry["ast_hash"] = h @@ -1387,15 +1709,21 @@ def _normalise_entry(entry): else: # Preserve semantic_hash only when content is unchanged entry["semantic_hash"] = prev.get("semantic_hash", "") if h == prev.get("ast_hash", "") else "" - manifest[f] = entry + manifest[key] = entry if root is not None: # Persist in portable form: forward-slash relative paths. Keys outside # ``root`` (out-of-tree symlinked corpora, --include sources) keep # their absolute form so the manifest round-trips on the saving # machine even when not every entry can be portably encoded. - manifest = {_to_relative_for_storage(k, root): v for k, v in manifest.items()} - Path(manifest_path).parent.mkdir(parents=True, exist_ok=True) - Path(manifest_path).write_text(json.dumps(manifest, indent=2), encoding="utf-8") + # NFC after relativize so on-disk keys match what load_manifest + # re-anchors and compares against (#2221). + manifest = {_nfc(_to_relative_for_storage(k, root)): v for k, v in manifest.items()} + else: + manifest = {_nfc(k): v for k, v in manifest.items()} + from graphify.paths import write_json_atomic + # Atomic write: a crash mid-write must not leave a truncated manifest that + # detect_incremental then fails to parse. + write_json_atomic(manifest_path, manifest, indent=2) def detect_incremental( @@ -1406,6 +1734,7 @@ def detect_incremental( google_workspace: bool | None = None, kind: str = "semantic", extra_excludes: list[str] | None = None, + gitignore: bool = True, ) -> dict: """Like detect(), but returns only new or modified files since the last run. @@ -1429,7 +1758,13 @@ def detect_incremental( runs. ``None`` (default) does not follow symlinked directories; callers must opt in explicitly, and resolved targets outside the scan root are skipped. """ - full = detect(root, follow_symlinks=follow_symlinks, google_workspace=google_workspace, extra_excludes=extra_excludes) + full = detect( + root, + follow_symlinks=follow_symlinks, + google_workspace=google_workspace, + extra_excludes=extra_excludes, + gitignore=gitignore, + ) # Pass ``root`` so a manifest written with relative keys (post-#777) is # re-anchored to the absolute form the rest of this function compares # against. Legacy absolute-keyed manifests pass through unchanged. @@ -1441,6 +1776,8 @@ def detect_incremental( full["new_files"] = full["files"] full["unchanged_files"] = {k: [] for k in full["files"]} full["new_total"] = full["total_files"] + full["deleted_files"] = [] + full["excluded_files"] = [] return full new_files: dict[str, list[str]] = {k: [] for k in full["files"]} @@ -1448,15 +1785,22 @@ def detect_incremental( for ftype, file_list in full["files"].items(): for f in file_list: - stored = manifest.get(f) + # Manifest keys are NFC; scan paths may arrive NFD (#2221). + stored = manifest.get(_nfc(f)) try: current_mtime = os.stat(_os_path(Path(f))).st_mtime except Exception: current_mtime = 0 - # Legacy manifest: plain float value — treat as ast_hash only + # Legacy manifest: plain float value stores only mtime. + # Compare with `!=` so backwards mtime motion (git checkout of an + # older commit, tarball restore, rsync --times) still triggers a + # re-extract; the previous `>` silently kept the stale cache and + # the graph drifted from disk (#1859). No stored hash means we + # cannot verify content — any mtime delta forces a re-extract, + # and the next save promotes the entry into the dict schema. if isinstance(stored, (int, float)): - changed = stored is None or current_mtime > stored + changed = current_mtime != stored elif isinstance(stored, dict): # Normalise legacy {mtime, hash} to new schema if "hash" in stored and "ast_hash" not in stored: @@ -1487,9 +1831,23 @@ def detect_incremental( else: unchanged_files[ftype].append(f) - # Files in manifest that no longer exist - their cached nodes are now ghost nodes - current_files = {f for flist in full["files"].values() for f in flist} - deleted_files = [f for f in manifest if f not in current_files] + # Manifest rows that left the corpus, split by disk existence (#1908): + # a row whose file is gone from DISK is a genuine deletion (its cached + # nodes are ghosts); a row whose file still exists but is out of the + # current scan was EXCLUDED (ignore rules / --exclude changed) and must + # not be reported as deleted. Mirrors the watch-side excluded-vs-deleted + # distinction (#1795). + current_files = {_nfc(f) for flist in full["files"].values() for f in flist} + deleted_files: list[str] = [] + excluded_files: list[str] = [] + for f in manifest: + if _nfc(f) in current_files: + continue + try: + alive = Path(f).exists() + except OSError: + alive = False + (excluded_files if alive else deleted_files).append(f) new_total = sum(len(v) for v in new_files.values()) full["incremental"] = True @@ -1497,4 +1855,5 @@ def detect_incremental( full["unchanged_files"] = unchanged_files full["new_total"] = new_total full["deleted_files"] = deleted_files + full["excluded_files"] = excluded_files return full diff --git a/graphify/diagnostics.py b/graphify/diagnostics.py index ff66fa958..fcb9a11cf 100644 --- a/graphify/diagnostics.py +++ b/graphify/diagnostics.py @@ -168,6 +168,14 @@ def diagnose_extraction( raw_edges = _edge_list(extraction) canonical_edges = [_canonical_edge(edge) for edge in raw_edges] + # Code-typed semantic nodes the extractor could not verify against the source + # it read (#1949): likely-inferred (or hallucinated) symbols surfaced from a + # document. Count them so the flag on graph.json nodes is actually surfaced. + unverified_node_count = sum( + 1 for n in extraction.get("nodes", []) + if isinstance(n, dict) and n.get("verification") == "unverified" + ) + exact_counts: Counter[str] = Counter(_exact_signature(edge) for edge in raw_edges) directed_pairs: Counter[tuple[str, str]] = Counter() undirected_pairs: Counter[tuple[str, str]] = Counter() @@ -239,6 +247,7 @@ def diagnose_extraction( return { "node_count": len(node_ids), + "unverified_node_count": unverified_node_count, "raw_edge_count": len(raw_edges), "non_object_edges": non_object_edges, "missing_endpoint_edges": missing_endpoint_edges, @@ -344,6 +353,7 @@ def format_diagnostic_report(summary: dict[str, Any]) -> str: "input_stage: provided JSON (normal graph.json is post-build)", f"effective_directed: {summary.get('effective_directed', '')}", f"nodes: {summary['node_count']}", + f"unverified_code_nodes: {summary.get('unverified_node_count', 0)}", f"raw_edges: {summary['raw_edge_count']}", f"valid_candidate_edges: {summary['valid_candidate_edges']}", f"missing_endpoint_edges: {summary['missing_endpoint_edges']}", diff --git a/graphify/export.py b/graphify/export.py index 36e4f9949..e1f2caa99 100644 --- a/graphify/export.py +++ b/graphify/export.py @@ -17,6 +17,8 @@ from graphify.analyze import _node_community_map from graphify.build import edge_data +from graphify.exporters.graphdb import push_to_falkordb, push_to_neo4j # noqa: E402,F401 + # Artifacts worth preserving across rebuilds (non-regenerable without LLM or curation). _BACKUP_ARTIFACTS = [ @@ -149,313 +151,9 @@ def _yaml_str(s: str) -> str: return "".join(out) -COMMUNITY_COLORS = [ - "#4E79A7", "#F28E2B", "#E15759", "#76B7B2", "#59A14F", - "#EDC948", "#B07AA1", "#FF9DA7", "#9C755F", "#BAB0AC", -] - -MAX_NODES_FOR_VIZ = 5_000 - +from graphify.exporters.base import COMMUNITY_COLORS # noqa: E402,F401 -def _viz_node_limit() -> int: - """Return the effective viz node limit, honoring GRAPHIFY_VIZ_NODE_LIMIT env var. - - Falls back to MAX_NODES_FOR_VIZ when the env var is unset, empty, or non-integer. - Set to 0 to disable HTML viz unconditionally (useful for CI runners). - """ - import os - raw = os.environ.get("GRAPHIFY_VIZ_NODE_LIMIT") - if raw is None or not raw.strip(): - return MAX_NODES_FOR_VIZ - try: - return int(raw) - except ValueError: - return MAX_NODES_FOR_VIZ - - -def _html_styles() -> str: - return """""" - - -def _hyperedge_script(hyperedges_json: str) -> str: - return f"""""" - - -def _html_script(nodes_json: str, edges_json: str, legend_json: str) -> str: - return f"""""" +from graphify.exporters.html import to_html # noqa: E402,F401 _CONFIDENCE_SCORE_DEFAULTS = {"EXTRACTED": 1.0, "INFERRED": 0.5, "AMBIGUOUS": 0.2} @@ -482,15 +180,97 @@ def _git_head() -> str | None: return None +# Sentinel: an existing graph.json is present and non-empty but cannot be parsed +# into a node count (corrupt, mid-write, or structurally wrong). The caller must +# fail CLOSED on this — the same way to_json's #479 guard refuses to overwrite +# such a file — because we cannot prove the new graph isn't a silent shrink. +MALFORMED_GRAPH = object() + + +def existing_graph_node_count(path: "str | Path"): + """Node count of an existing graph.json. + + Returns: + - an ``int`` node count when the file parses; + - ``None`` when there is verifiably nothing to protect — absent, empty, or + over the size cap (matching how :func:`to_json` lets the new graph + replace an empty/oversized file); + - :data:`MALFORMED_GRAPH` when the file is present and non-empty but + unparseable — the caller must treat this as fail-closed (refuse to + overwrite), mirroring to_json's #479 handling of a corrupt/mid-write file. + + The raw ``--no-cluster`` write path uses this to apply the same #479 shrink + guard that :func:`to_json` applies inline for the clustered path. + """ + p = Path(path) + if not p.exists(): + return None + from graphify.security import check_graph_file_size_cap + try: + check_graph_file_size_cap(p) + except Exception: + # Oversized: reading it to compare would be the DoS the cap guards against. + return None + try: + raw = p.read_text(encoding="utf-8") + except Exception: + # Present but unreadable: fail closed if it has bytes, else nothing to lose. + try: + return MALFORMED_GRAPH if p.stat().st_size > 0 else None + except Exception: + return None + if not raw.strip(): + return None + try: + data = json.loads(raw) + except Exception: + return MALFORMED_GRAPH + nodes = data.get("nodes") if isinstance(data, dict) else None + return len(nodes) if isinstance(nodes, list) else MALFORMED_GRAPH + + def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str, *, force: bool = False, built_at_commit: str | None = None, community_labels: dict[int, str] | None = None) -> bool: # Safety check: refuse to silently shrink an existing graph (#479) existing_path = Path(output_path) if not force and existing_path.exists(): + from graphify.security import check_graph_file_size_cap try: - from graphify.security import check_graph_file_size_cap check_graph_file_size_cap(existing_path) - existing_data = json.loads(existing_path.read_text(encoding="utf-8")) - existing_n = len(existing_data.get("nodes", [])) + except Exception: + # Existing graph.json trips the size cap; reading it to compare would + # be the very DoS the cap guards against. Can't verify — let the new + # graph replace the oversized file. + oversized = True + else: + oversized = False + if not oversized: + try: + raw = existing_path.read_text(encoding="utf-8") + except Exception: + raw = "" + if not raw.strip(): + # Empty/whitespace existing file (e.g. a freshly touched path): + # no nodes to lose, so any new graph is a growth — proceed. + existing_n = 0 + else: + try: + existing_data = json.loads(raw) + existing_n = len(existing_data.get("nodes", [])) + except Exception as exc: + # Non-empty but unparseable existing graph (corrupt or a + # mid-write): we cannot verify the new graph is not a silent + # shrink. Fail SAFE — refuse rather than overwrite. A + # fail-OPEN here (the prior behavior) is the silent data-loss + # path #479 exists to prevent: a transiently unreadable + # graph.json would let a partial rebuild clobber a good one. + import sys as _sys + print( + f"[graphify] WARNING: existing {existing_path} could not be " + f"read to verify the new graph is not smaller ({exc}). " + f"Refusing to overwrite; pass force=True to override.", + file=_sys.stderr, + ) + return False new_n = G.number_of_nodes() if new_n < existing_n: import sys as _sys @@ -505,8 +285,6 @@ def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str, *, file=_sys.stderr, ) return False - except Exception: - pass # unreadable existing file — proceed with write node_community = _node_community_map(communities) _labels: dict[int, str] = {int(k): v for k, v in (community_labels or {}).items()} @@ -537,8 +315,9 @@ def to_json(G: nx.Graph, communities: dict[int, list[str]], output_path: str, *, commit = built_at_commit if built_at_commit is not None else _git_head() if commit: data["built_at_commit"] = commit - with open(output_path, "w", encoding="utf-8") as f: # nosec - json.dump(data, f, indent=2) + from graphify.paths import write_json_atomic + # Atomic write: a crash/ENOSPC mid-write must not truncate a good graph.json. + write_json_atomic(output_path, data, indent=2) return True @@ -629,244 +408,6 @@ def to_cypher(G: nx.Graph, output_path: str) -> None: f.write("\n".join(lines)) -def to_html( - G: nx.Graph, - communities: dict[int, list[str]], - output_path: str, - community_labels: dict[int, str] | None = None, - member_counts: dict[int, int] | None = None, - node_limit: int | None = None, - learning_overlay: dict | None = None, -) -> None: - """Generate an interactive vis.js HTML visualization of the graph. - - Features: node size by degree, click-to-inspect panel, search box, - community filter, physics clustering by community, confidence-styled edges. - Raises ValueError if graph exceeds MAX_NODES_FOR_VIZ. - - If member_counts is provided (aggregated community view), node sizes are - based on community member counts rather than graph degree. - - If node_limit is set and the graph exceeds it, automatically builds an - aggregated community-level meta-graph instead of raising ValueError. - """ - limit = node_limit if node_limit is not None else _viz_node_limit() - if G.number_of_nodes() > limit: - if node_limit is not None: - # Build aggregated community meta-graph - from collections import Counter as _Counter - import networkx as _nx - print(f"Graph has {G.number_of_nodes()} nodes (above {limit} limit). Building aggregated community view...") - node_to_community = {nid: cid for cid, members in communities.items() for nid in members} - meta = _nx.Graph() - for cid, members in communities.items(): - meta.add_node(str(cid), label=(community_labels or {}).get(cid, f"Community {cid}")) - edge_counts = _Counter() - for u, v in G.edges(): - cu, cv = node_to_community.get(u), node_to_community.get(v) - if cu is not None and cv is not None and cu != cv: - edge_counts[(min(cu, cv), max(cu, cv))] += 1 - for (cu, cv), w in edge_counts.items(): - meta.add_edge(str(cu), str(cv), weight=w, - relation=f"{w} cross-community edges", confidence="AGGREGATED") - if meta.number_of_nodes() <= 1: - print("Single community - aggregated view not useful. Skipping graph.html.") - return - meta_communities = {cid: [str(cid)] for cid in communities} - mc = {cid: len(members) for cid, members in communities.items()} - # Remap hyperedges from semantic node IDs to community IDs - raw_hyperedges = G.graph.get("hyperedges", []) - if raw_hyperedges: - remapped = [] - for he in raw_hyperedges: - he_members = he.get("nodes", []) - comm_ids, seen = [], set() - for nid in he_members: - c = node_to_community.get(nid) - if c is None: - continue - s = str(c) - if s in seen: - continue - seen.add(s) - comm_ids.append(s) - if len(comm_ids) < 2: - continue - remapped.append({ - "id": he.get("id", ""), - "label": he.get("label") or he.get("relation", "").replace("_", " "), - "nodes": comm_ids, - }) - meta.graph["hyperedges"] = remapped - to_html(meta, meta_communities, output_path, - community_labels=community_labels, member_counts=mc) - print(f"graph.html written (aggregated: {meta.number_of_nodes()} community nodes, {meta.number_of_edges()} cross-community edges)") - print("Tip: run with --obsidian for full node-level detail.") - return - raise ValueError( - f"Graph has {G.number_of_nodes()} nodes - too large for HTML viz " - f"(limit: {limit}). Use --no-viz, raise GRAPHIFY_VIZ_NODE_LIMIT, " - f"or reduce input size." - ) - - node_community = _node_community_map(communities) - degree = dict(G.degree()) - max_deg = max(degree.values(), default=1) or 1 - max_mc = (max(member_counts.values(), default=1) or 1) if member_counts else 1 - - # Work-memory overlay (derived sidecar). When not passed explicitly, load it - # best-effort from the sibling .graphify_learning.json next to the output - # graph.html (which lives beside graph.json). Empty/missing => no learning - # fields, so the un-annotated render is byte-identical to pre-feature. - if learning_overlay is None: - learning_overlay = {} - try: - from graphify.reflect import load_learning_overlay as _llo - learning_overlay = _llo(Path(output_path)) - except Exception: - learning_overlay = {} - # Status -> ring color. preferred=green, contested=amber. Tentative gets no - # ring (it's not yet trustworthy enough to highlight in the map). - _RING = {"preferred": "#22c55e", "contested": "#f59e0b"} - - # Build nodes list for vis.js - vis_nodes = [] - for node_id, data in G.nodes(data=True): - cid = node_community.get(node_id, 0) - color = COMMUNITY_COLORS[cid % len(COMMUNITY_COLORS)] - label = sanitize_label(data.get("label", node_id)) - deg = degree.get(node_id, 1) - if member_counts: - mc = member_counts.get(cid, 1) - size = 10 + 30 * (mc / max_mc) - font_size = 12 - else: - size = 10 + 30 * (deg / max_deg) - # Only show label for high-degree nodes by default; others show on hover - font_size = 12 if deg >= max_deg * 0.15 else 0 - node = { - "id": node_id, - "label": label, - "color": {"background": color, "border": color, "highlight": {"background": "#ffffff", "border": color}}, - "size": round(size, 1), - "font": {"size": font_size, "color": "#ffffff"}, - "title": _html.escape(label), - "community": cid, - "community_name": sanitize_label((community_labels or {}).get(cid, f"Community {cid}")), - "source_file": sanitize_label(str(data.get("source_file") or "")), - "file_type": data.get("file_type", ""), - "degree": deg, - } - # Conditional learning fields — only present for annotated nodes, so - # un-annotated output keeps the exact pre-feature node dict shape. - entry = learning_overlay.get(str(node_id)) if learning_overlay else None - if entry: - status = sanitize_label(str(entry.get("status", ""))) - stale = bool(entry.get("stale")) - node["learning_status"] = status - node["learning_stale"] = stale - ring = _RING.get(status) - if ring: - # Status-colored ring via the border; stale => desaturated + - # dashed (vis.js supports per-node `shapeProperties.borderDashes`). - if stale: - ring = "#9ca3af" - node["shapeProperties"] = {"borderDashes": [4, 4]} - node["borderWidth"] = 3 - node["color"] = { - "background": color, "border": ring, - "highlight": {"background": "#ffffff", "border": ring}, - } - # Lesson line appended to the hover title. - if status == "contested": - lesson = f"Lesson: contested (useful {entry.get('uses', 0)} / dead-end {entry.get('neg', 0)})" - elif status == "preferred": - lesson = f"Lesson: preferred source ({entry.get('uses', 0)} useful, score={entry.get('score', 0)})" - else: - lesson = f"Lesson: {status} ({entry.get('uses', 0)} useful)" - if stale: - lesson += " [code changed — re-verify]" - node["title"] = _html.escape(label) + "\n" + _html.escape(sanitize_label(lesson)) - vis_nodes.append(node) - - # Build edges list. Restore original edge direction from _src/_tgt - # (stashed by build.py for exactly this reason): undirected NetworkX - # canonicalizes endpoint order, which would otherwise flip the arrow - # for `calls` and `rationale_for` in the rendered graph (#563). - vis_edges = [] - for u, v, data in G.edges(data=True): - confidence = data.get("confidence", "EXTRACTED") - relation = data.get("relation", "") - true_src = data.get("_src", u) - true_tgt = data.get("_tgt", v) - vis_edges.append({ - "from": true_src, - "to": true_tgt, - "label": relation, - "title": _html.escape(f"{relation} [{confidence}]"), - "dashes": confidence != "EXTRACTED", - "width": 2 if confidence == "EXTRACTED" else 1, - "color": {"opacity": 0.7 if confidence == "EXTRACTED" else 0.35}, - "confidence": confidence, - }) - - # Build community legend data - legend_data = [] - for cid in sorted((community_labels or {}).keys()): - color = COMMUNITY_COLORS[cid % len(COMMUNITY_COLORS)] - lbl = _html.escape(sanitize_label((community_labels or {}).get(cid, f"Community {cid}"))) - n = member_counts.get(cid, len(communities.get(cid, []))) if member_counts else len(communities.get(cid, [])) - legend_data.append({"cid": cid, "color": color, "label": lbl, "count": n}) - - # Escape sequences so embedded JSON cannot break out of the script tag - def _js_safe(obj) -> str: - return json.dumps(obj).replace(" - - - -graphify - {title} - -{_html_styles()} - - -
- -{_html_script(nodes_json, edges_json, legend_json)} -{_hyperedge_script(hyperedges_json)} - -""" - - Path(output_path).write_text(html, encoding="utf-8") # nosec - - # Keep backward-compatible alias - skill.md calls generate_html generate_html = to_html @@ -1199,6 +740,28 @@ def _community_name(cid) -> str: } _owned_write(".obsidian/graph.json", json.dumps(graph_config, indent=2)) + # #1896: prune notes for nodes that dropped out of the graph. Only files the + # manifest says graphify owns are candidates, and anything written or skipped + # this run is excluded — so a user's own note is never touched (foreign files + # land in _skipped, never _owned). Guard each path to stay inside the vault in + # case a corrupt/hostile manifest contains `../` entries. + stale = _owned - set(_written) - set(_skipped) + pruned = 0 + for rel_name in sorted(stale): + target = (out / rel_name).resolve() + if out.resolve() not in target.parents: + continue + try: + target.unlink(missing_ok=True) + pruned += 1 + except OSError: + pass + if pruned: + print( + f"[graphify] pruned {pruned} note(s) for nodes no longer in the graph", + file=sys.stderr, + ) + # Persist the manifest of files graphify owns, so a re-run can safely update its # own notes while still refusing to touch the user's. Warn (once, aggregated) # about anything skipped to avoid clobbering a pre-existing file. @@ -1397,174 +960,6 @@ def safe_name(label: str) -> str: Path(output_path).write_text(json.dumps(canvas_data, indent=2), encoding="utf-8") # nosec -def push_to_neo4j( - G: nx.Graph, - uri: str, - user: str, - password: str, - communities: dict[int, list[str]] | None = None, -) -> dict[str, int]: - """Push graph directly to a running Neo4j instance via the Python driver. - - Requires: pip install neo4j - - Uses MERGE so re-running is safe - nodes and edges are upserted, not duplicated. - Returns a dict with counts of nodes and edges pushed. - """ - try: - from neo4j import GraphDatabase - except ImportError as e: - raise ImportError( - "neo4j driver not installed. Run: pip install neo4j" - ) from e - - node_community = _node_community_map(communities) if communities else {} - - def _safe_rel(relation: str) -> str: - return re.sub(r"[^A-Z0-9_]", "_", relation.upper().replace(" ", "_").replace("-", "_")) or "RELATED_TO" - - def _safe_label(label: str) -> str: - """Sanitize a Neo4j node label to prevent Cypher injection.""" - sanitized = re.sub(r"[^A-Za-z0-9_]", "", label) - return sanitized if sanitized else "Entity" - - driver = GraphDatabase.driver(uri, auth=(user, password)) - nodes_pushed = 0 - edges_pushed = 0 - - with driver.session() as session: - for node_id, data in G.nodes(data=True): - props = { - k: v for k, v in data.items() - if isinstance(v, (str, int, float, bool)) and not k.startswith("_") - } - props["id"] = node_id - cid = node_community.get(node_id) - if cid is not None: - props["community"] = cid - ftype = _safe_label(data.get("file_type", "Entity").capitalize()) - session.run( - f"MERGE (n:{ftype} {{id: $id}}) SET n += $props", - id=node_id, - props=props, - ) - nodes_pushed += 1 - - for u, v, data in G.edges(data=True): - rel = _safe_rel(data.get("relation", "RELATED_TO")) - props = { - k: v for k, v in data.items() - if isinstance(v, (str, int, float, bool)) and not k.startswith("_") - } - session.run( - f"MATCH (a {{id: $src}}), (b {{id: $tgt}}) " - f"MERGE (a)-[r:{rel}]->(b) SET r += $props", - src=u, - tgt=v, - props=props, - ) - edges_pushed += 1 - - driver.close() - return {"nodes": nodes_pushed, "edges": edges_pushed} - - -def push_to_falkordb( - G: nx.Graph, - uri: str, - user: str | None = None, - password: str | None = None, - communities: dict[int, list[str]] | None = None, - graph_name: str = "graphify", -) -> dict[str, int]: - """Push graph directly to a running FalkorDB instance via the Python SDK. - - Requires: pip install falkordb - - FalkorDB is OpenCypher-compatible, so the MERGE/SET upsert queries are - identical to push_to_neo4j. Differences from the Neo4j path: - - connects with FalkorDB(host, port, username, password) instead of a bolt - driver; only the host/port are read from the URI, so the scheme is - informational - "falkordb://localhost:6379", "redis://localhost:6379" - and a bare "localhost:6379" are all equivalent (default port 6379). - - a named graph is selected via db.select_graph(graph_name) (default - "graphify"); FalkorDB keys each graph by name in the same instance. - - queries run via graph.query(cypher, params) - there is no session object. - - auth is optional (FalkorDB runs without credentials by default), so user - and password may be None. - - no APOC: the Neo4j path does not use APOC either, so nothing to port. - - Uses MERGE so re-running is safe - nodes and edges are upserted, not - duplicated. Returns a dict with counts of nodes and edges pushed. - """ - try: - from falkordb import FalkorDB - except ImportError as e: - raise ImportError( - "falkordb SDK not installed. Run: pip install falkordb" - ) from e - - from urllib.parse import urlparse - - node_community = _node_community_map(communities) if communities else {} - - def _safe_rel(relation: str) -> str: - return re.sub(r"[^A-Z0-9_]", "_", relation.upper().replace(" ", "_").replace("-", "_")) or "RELATED_TO" - - def _safe_label(label: str) -> str: - """Sanitize a FalkorDB node label to prevent Cypher injection.""" - sanitized = re.sub(r"[^A-Za-z0-9_]", "", label) - return sanitized if sanitized else "Entity" - - parsed = urlparse(uri if "://" in uri else f"redis://{uri}") - # FalkorDB auth is optional. Only send credentials when a password is - # provided; otherwise connect anonymously and ignore any bolt-style default - # username (e.g. Neo4j's "neo4j"), which FalkorDB rejects as an unknown ACL - # user. Credentials embedded in the URI take precedence over the args. - connect_user = parsed.username or (user if password else None) - connect_password = parsed.password or (password or None) - db = FalkorDB( - host=parsed.hostname or "localhost", - port=parsed.port or 6379, - username=connect_user, - password=connect_password, - ) - graph = db.select_graph(graph_name) - nodes_pushed = 0 - edges_pushed = 0 - - for node_id, data in G.nodes(data=True): - props = { - k: v for k, v in data.items() - if isinstance(v, (str, int, float, bool)) and not k.startswith("_") - } - props["id"] = node_id - cid = node_community.get(node_id) - if cid is not None: - props["community"] = cid - ftype = _safe_label(data.get("file_type", "Entity").capitalize()) - graph.query( - f"MERGE (n:{ftype} {{id: $id}}) SET n += $props", - {"id": node_id, "props": props}, - ) - nodes_pushed += 1 - - for u, v, data in G.edges(data=True): - rel = _safe_rel(data.get("relation", "RELATED_TO")) - props = { - k: v for k, v in data.items() - if isinstance(v, (str, int, float, bool)) and not k.startswith("_") - } - graph.query( - f"MATCH (a {{id: $src}}), (b {{id: $tgt}}) " - f"MERGE (a)-[r:{rel}]->(b) SET r += $props", - {"src": u, "tgt": v, "props": props}, - ) - edges_pushed += 1 - - return {"nodes": nodes_pushed, "edges": edges_pushed} - - def to_graphml( G: nx.Graph, communities: dict[int, list[str]], @@ -1588,16 +983,44 @@ def to_graphml( for _, _, attrs in H.edges(data=True): for k in [k for k in attrs if k.startswith("_")]: del attrs[k] - # nx.write_graphml raises ValueError on None attribute values; replace with "". + # nx.write_graphml only accepts scalar attribute values: None raises, and a + # dict/list value (e.g. a per-node `metadata` dict, or the graph-level + # `hyperedges` list set by attach_hyperedges()) raises + # "GraphML does not support type as data values" (#1831). + # Coerce None -> "" and non-scalars -> a JSON string, across all three scopes. + def _graphml_safe(val): + if val is None: + return "" + if isinstance(val, bool) or isinstance(val, (int, float, str)): + return val # GraphML-native scalars pass through unchanged + try: + return json.dumps(val, default=str, sort_keys=True) + except (TypeError, ValueError): + return str(val) + + for key, val in list(H.graph.items()): + H.graph[key] = _graphml_safe(val) for node_id in H.nodes(): for key, val in list(H.nodes[node_id].items()): - if val is None: - H.nodes[node_id][key] = "" + H.nodes[node_id][key] = _graphml_safe(val) for u, v in H.edges(): for key, val in list(H.edges[u, v].items()): - if val is None: - H.edges[u, v][key] = "" - nx.write_graphml(H, output_path) + H.edges[u, v][key] = _graphml_safe(val) + + # Write atomically: a mid-serialization error otherwise leaves a 0-byte + # .graphml on disk that downstream tooling mistakes for a completed export + # (#1831). Write to a sibling temp file, then replace on success. + out = Path(output_path) + tmp = out.with_name(out.name + ".tmp") + try: + nx.write_graphml(H, str(tmp)) + os.replace(str(tmp), str(out)) + finally: + if tmp.exists(): + try: + tmp.unlink() + except OSError: + pass def to_svg( diff --git a/graphify/exporters/__init__.py b/graphify/exporters/__init__.py new file mode 100644 index 000000000..41e1aeff9 --- /dev/null +++ b/graphify/exporters/__init__.py @@ -0,0 +1 @@ +"""exporters package.""" diff --git a/graphify/exporters/base.py b/graphify/exporters/base.py new file mode 100644 index 000000000..9c9419646 --- /dev/null +++ b/graphify/exporters/base.py @@ -0,0 +1,14 @@ +"""Shared constants/helpers for the graphify exporters package. + +Symbols used by more than one exporter live here so each exporter module can be +split out of graphify/export.py without a circular import (export.py and the +per-format modules both import from here, never from each other). +""" +from __future__ import annotations + +# Categorical palette for community coloring, shared by the HTML, SVG, and +# Obsidian exporters. Moved verbatim from graphify/export.py. +COMMUNITY_COLORS = [ + "#4E79A7", "#F28E2B", "#E15759", "#76B7B2", "#59A14F", + "#EDC948", "#B07AA1", "#FF9DA7", "#9C755F", "#BAB0AC", +] diff --git a/graphify/exporters/graphdb.py b/graphify/exporters/graphdb.py new file mode 100644 index 000000000..14c47f0d5 --- /dev/null +++ b/graphify/exporters/graphdb.py @@ -0,0 +1,173 @@ +"""graphdb — moved verbatim from graphify/export.py.""" +from __future__ import annotations + +from graphify.analyze import _node_community_map +import networkx as nx +import re + + +def push_to_neo4j( + G: nx.Graph, + uri: str, + user: str, + password: str, + communities: dict[int, list[str]] | None = None, +) -> dict[str, int]: + """Push graph directly to a running Neo4j instance via the Python driver. + + Requires: pip install neo4j + + Uses MERGE so re-running is safe - nodes and edges are upserted, not duplicated. + Returns a dict with counts of nodes and edges pushed. + """ + try: + from neo4j import GraphDatabase + except ImportError as e: + raise ImportError( + "neo4j driver not installed. Run: pip install neo4j" + ) from e + + node_community = _node_community_map(communities) if communities else {} + + def _safe_rel(relation: str) -> str: + return re.sub(r"[^A-Z0-9_]", "_", relation.upper().replace(" ", "_").replace("-", "_")) or "RELATED_TO" + + def _safe_label(label: str) -> str: + """Sanitize a Neo4j node label to prevent Cypher injection.""" + sanitized = re.sub(r"[^A-Za-z0-9_]", "", label) + return sanitized if sanitized else "Entity" + + driver = GraphDatabase.driver(uri, auth=(user, password)) + nodes_pushed = 0 + edges_pushed = 0 + + with driver.session() as session: + for node_id, data in G.nodes(data=True): + props = { + k: v for k, v in data.items() + if isinstance(v, (str, int, float, bool)) and not k.startswith("_") + } + props["id"] = node_id + cid = node_community.get(node_id) + if cid is not None: + props["community"] = cid + ftype = _safe_label(data.get("file_type", "Entity").capitalize()) + session.run( + f"MERGE (n:{ftype} {{id: $id}}) SET n += $props", + id=node_id, + props=props, + ) + nodes_pushed += 1 + + for u, v, data in G.edges(data=True): + rel = _safe_rel(data.get("relation", "RELATED_TO")) + props = { + k: v for k, v in data.items() + if isinstance(v, (str, int, float, bool)) and not k.startswith("_") + } + session.run( + f"MATCH (a {{id: $src}}), (b {{id: $tgt}}) " + f"MERGE (a)-[r:{rel}]->(b) SET r += $props", + src=u, + tgt=v, + props=props, + ) + edges_pushed += 1 + + driver.close() + return {"nodes": nodes_pushed, "edges": edges_pushed} + +def push_to_falkordb( + G: nx.Graph, + uri: str, + user: str | None = None, + password: str | None = None, + communities: dict[int, list[str]] | None = None, + graph_name: str = "graphify", +) -> dict[str, int]: + """Push graph directly to a running FalkorDB instance via the Python SDK. + + Requires: pip install falkordb + + FalkorDB is OpenCypher-compatible, so the MERGE/SET upsert queries are + identical to push_to_neo4j. Differences from the Neo4j path: + - connects with FalkorDB(host, port, username, password) instead of a bolt + driver; only the host/port are read from the URI, so the scheme is + informational - "falkordb://localhost:6379", "redis://localhost:6379" + and a bare "localhost:6379" are all equivalent (default port 6379). + - a named graph is selected via db.select_graph(graph_name) (default + "graphify"); FalkorDB keys each graph by name in the same instance. + - queries run via graph.query(cypher, params) - there is no session object. + - auth is optional (FalkorDB runs without credentials by default), so user + and password may be None. + - no APOC: the Neo4j path does not use APOC either, so nothing to port. + + Uses MERGE so re-running is safe - nodes and edges are upserted, not + duplicated. Returns a dict with counts of nodes and edges pushed. + """ + try: + from falkordb import FalkorDB + except ImportError as e: + raise ImportError( + "falkordb SDK not installed. Run: pip install falkordb" + ) from e + + from urllib.parse import urlparse + + node_community = _node_community_map(communities) if communities else {} + + def _safe_rel(relation: str) -> str: + return re.sub(r"[^A-Z0-9_]", "_", relation.upper().replace(" ", "_").replace("-", "_")) or "RELATED_TO" + + def _safe_label(label: str) -> str: + """Sanitize a FalkorDB node label to prevent Cypher injection.""" + sanitized = re.sub(r"[^A-Za-z0-9_]", "", label) + return sanitized if sanitized else "Entity" + + parsed = urlparse(uri if "://" in uri else f"redis://{uri}") + # FalkorDB auth is optional. Only send credentials when a password is + # provided; otherwise connect anonymously and ignore any bolt-style default + # username (e.g. Neo4j's "neo4j"), which FalkorDB rejects as an unknown ACL + # user. Credentials embedded in the URI take precedence over the args. + connect_user = parsed.username or (user if password else None) + connect_password = parsed.password or (password or None) + db = FalkorDB( + host=parsed.hostname or "localhost", + port=parsed.port or 6379, + username=connect_user, + password=connect_password, + ) + graph = db.select_graph(graph_name) + nodes_pushed = 0 + edges_pushed = 0 + + for node_id, data in G.nodes(data=True): + props = { + k: v for k, v in data.items() + if isinstance(v, (str, int, float, bool)) and not k.startswith("_") + } + props["id"] = node_id + cid = node_community.get(node_id) + if cid is not None: + props["community"] = cid + ftype = _safe_label(data.get("file_type", "Entity").capitalize()) + graph.query( + f"MERGE (n:{ftype} {{id: $id}}) SET n += $props", + {"id": node_id, "props": props}, + ) + nodes_pushed += 1 + + for u, v, data in G.edges(data=True): + rel = _safe_rel(data.get("relation", "RELATED_TO")) + props = { + k: v for k, v in data.items() + if isinstance(v, (str, int, float, bool)) and not k.startswith("_") + } + graph.query( + f"MATCH (a {{id: $src}}), (b {{id: $tgt}}) " + f"MERGE (a)-[r:{rel}]->(b) SET r += $props", + {"src": u, "tgt": v, "props": props}, + ) + edges_pushed += 1 + + return {"nodes": nodes_pushed, "edges": edges_pushed} diff --git a/graphify/exporters/html.py b/graphify/exporters/html.py new file mode 100644 index 000000000..59c0e52e3 --- /dev/null +++ b/graphify/exporters/html.py @@ -0,0 +1,560 @@ +"""html — moved verbatim from graphify/export.py.""" +from __future__ import annotations + +from graphify.exporters.base import COMMUNITY_COLORS # noqa: E402,F401 +from pathlib import Path +import html as _html +from graphify.analyze import _node_community_map +import json +import networkx as nx +from graphify.security import sanitize_label + + +MAX_NODES_FOR_VIZ = 5_000 + +def _viz_node_limit() -> int: + """Return the effective viz node limit, honoring GRAPHIFY_VIZ_NODE_LIMIT env var. + + Falls back to MAX_NODES_FOR_VIZ when the env var is unset, empty, or non-integer. + Set to 0 to disable HTML viz unconditionally (useful for CI runners). + """ + import os + raw = os.environ.get("GRAPHIFY_VIZ_NODE_LIMIT") + if raw is None or not raw.strip(): + return MAX_NODES_FOR_VIZ + try: + return int(raw) + except ValueError: + return MAX_NODES_FOR_VIZ + +def _html_styles() -> str: + return """""" + +def _hyperedge_script(hyperedges_json: str) -> str: + return f"""""" + +def _html_script(nodes_json: str, edges_json: str, legend_json: str) -> str: + return f"""""" + +def to_html( + G: nx.Graph, + communities: dict[int, list[str]], + output_path: str, + community_labels: dict[int, str] | None = None, + member_counts: dict[int, int] | None = None, + node_limit: int | None = None, + learning_overlay: dict | None = None, +) -> None: + """Generate an interactive vis.js HTML visualization of the graph. + + Features: node size by degree, click-to-inspect panel, search box, + community filter, physics clustering by community, confidence-styled edges. + Raises ValueError if graph exceeds MAX_NODES_FOR_VIZ. + + If member_counts is provided (aggregated community view), node sizes are + based on community member counts rather than graph degree. + + If node_limit is set and the graph exceeds it, automatically builds an + aggregated community-level meta-graph instead of raising ValueError. + """ + limit = node_limit if node_limit is not None else _viz_node_limit() + if G.number_of_nodes() > limit: + if node_limit is not None: + # Build aggregated community meta-graph + from collections import Counter as _Counter + import networkx as _nx + print(f"Graph has {G.number_of_nodes()} nodes (above {limit} limit). Building aggregated community view...") + node_to_community = {nid: cid for cid, members in communities.items() for nid in members} + meta = _nx.Graph() + for cid, members in communities.items(): + meta.add_node(str(cid), label=(community_labels or {}).get(cid, f"Community {cid}")) + edge_counts = _Counter() + for u, v in G.edges(): + cu, cv = node_to_community.get(u), node_to_community.get(v) + if cu is not None and cv is not None and cu != cv: + edge_counts[(min(cu, cv), max(cu, cv))] += 1 + for (cu, cv), w in edge_counts.items(): + meta.add_edge(str(cu), str(cv), weight=w, + relation=f"{w} cross-community edges", confidence="AGGREGATED") + if meta.number_of_nodes() <= 1: + print("Single community - aggregated view not useful. Skipping graph.html.") + return + meta_communities = {cid: [str(cid)] for cid in communities} + mc = {cid: len(members) for cid, members in communities.items()} + # Remap hyperedges from semantic node IDs to community IDs + raw_hyperedges = G.graph.get("hyperedges", []) + if raw_hyperedges: + remapped = [] + for he in raw_hyperedges: + he_members = he.get("nodes", []) + comm_ids, seen = [], set() + for nid in he_members: + c = node_to_community.get(nid) + if c is None: + continue + s = str(c) + if s in seen: + continue + seen.add(s) + comm_ids.append(s) + if len(comm_ids) < 2: + continue + remapped.append({ + "id": he.get("id", ""), + "label": he.get("label") or he.get("relation", "").replace("_", " "), + "nodes": comm_ids, + }) + meta.graph["hyperedges"] = remapped + to_html(meta, meta_communities, output_path, + community_labels=community_labels, member_counts=mc) + print(f"graph.html written (aggregated: {meta.number_of_nodes()} community nodes, {meta.number_of_edges()} cross-community edges)") + print("Tip: run with --obsidian for full node-level detail.") + return + raise ValueError( + f"Graph has {G.number_of_nodes()} nodes - too large for HTML viz " + f"(limit: {limit}). Use --no-viz, raise GRAPHIFY_VIZ_NODE_LIMIT, " + f"or reduce input size." + ) + + node_community = _node_community_map(communities) + degree = dict(G.degree()) + max_deg = max(degree.values(), default=1) or 1 + max_mc = (max(member_counts.values(), default=1) or 1) if member_counts else 1 + + # Work-memory overlay (derived sidecar). When not passed explicitly, load it + # best-effort from the sibling .graphify_learning.json next to the output + # graph.html (which lives beside graph.json). Empty/missing => no learning + # fields, so the un-annotated render is byte-identical to pre-feature. + if learning_overlay is None: + learning_overlay = {} + try: + from graphify.reflect import load_learning_overlay as _llo + learning_overlay = _llo(Path(output_path)) + except Exception: + learning_overlay = {} + # Status -> ring color. preferred=green, contested=amber. Tentative gets no + # ring (it's not yet trustworthy enough to highlight in the map). + _RING = {"preferred": "#22c55e", "contested": "#f59e0b"} + + # Build nodes list for vis.js + vis_nodes = [] + for node_id, data in G.nodes(data=True): + cid = node_community.get(node_id, 0) + color = COMMUNITY_COLORS[cid % len(COMMUNITY_COLORS)] + label = sanitize_label(data.get("label", node_id)) + deg = degree.get(node_id, 1) + if member_counts: + mc = member_counts.get(cid, 1) + size = 10 + 30 * (mc / max_mc) + font_size = 12 + else: + size = 10 + 30 * (deg / max_deg) + # Only show label for high-degree nodes by default; others show on hover + font_size = 12 if deg >= max_deg * 0.15 else 0 + node = { + "id": node_id, + "label": label, + "color": {"background": color, "border": color, "highlight": {"background": "#ffffff", "border": color}}, + "size": round(size, 1), + "font": {"size": font_size, "color": "#ffffff"}, + "title": _html.escape(label), + "community": cid, + "community_name": sanitize_label((community_labels or {}).get(cid, f"Community {cid}")), + "source_file": sanitize_label(str(data.get("source_file") or "")), + "file_type": data.get("file_type", ""), + "degree": deg, + } + # Conditional learning fields — only present for annotated nodes, so + # un-annotated output keeps the exact pre-feature node dict shape. + entry = learning_overlay.get(str(node_id)) if learning_overlay else None + if entry: + status = sanitize_label(str(entry.get("status", ""))) + stale = bool(entry.get("stale")) + node["learning_status"] = status + node["learning_stale"] = stale + ring = _RING.get(status) + if ring: + # Status-colored ring via the border; stale => desaturated + + # dashed (vis.js supports per-node `shapeProperties.borderDashes`). + if stale: + ring = "#9ca3af" + node["shapeProperties"] = {"borderDashes": [4, 4]} + node["borderWidth"] = 3 + node["color"] = { + "background": color, "border": ring, + "highlight": {"background": "#ffffff", "border": ring}, + } + # Lesson line appended to the hover title. + if status == "contested": + lesson = f"Lesson: contested (useful {entry.get('uses', 0)} / dead-end {entry.get('neg', 0)})" + elif status == "preferred": + lesson = f"Lesson: preferred source ({entry.get('uses', 0)} useful, score={entry.get('score', 0)})" + else: + lesson = f"Lesson: {status} ({entry.get('uses', 0)} useful)" + if stale: + lesson += " [code changed — re-verify]" + node["title"] = _html.escape(label) + "\n" + _html.escape(sanitize_label(lesson)) + vis_nodes.append(node) + + # Build edges list. Restore original edge direction from _src/_tgt + # (stashed by build.py for exactly this reason): undirected NetworkX + # canonicalizes endpoint order, which would otherwise flip the arrow + # for `calls` and `rationale_for` in the rendered graph (#563). + vis_edges = [] + for u, v, data in G.edges(data=True): + confidence = data.get("confidence", "EXTRACTED") + relation = data.get("relation", "") + true_src = data.get("_src", u) + true_tgt = data.get("_tgt", v) + vis_edges.append({ + "from": true_src, + "to": true_tgt, + "label": relation, + "title": _html.escape(f"{relation} [{confidence}]"), + "dashes": confidence != "EXTRACTED", + "width": 2 if confidence == "EXTRACTED" else 1, + "color": {"opacity": 0.7 if confidence == "EXTRACTED" else 0.35}, + "confidence": confidence, + }) + + # Build community legend data + legend_data = [] + for cid in sorted((community_labels or {}).keys()): + color = COMMUNITY_COLORS[cid % len(COMMUNITY_COLORS)] + lbl = _html.escape(sanitize_label((community_labels or {}).get(cid, f"Community {cid}"))) + n = member_counts.get(cid, len(communities.get(cid, []))) if member_counts else len(communities.get(cid, [])) + legend_data.append({"cid": cid, "color": color, "label": lbl, "count": n}) + + # Escape sequences so embedded JSON cannot break out of the script tag + def _js_safe(obj) -> str: + return json.dumps(obj).replace(" + + + +graphify - {title} + +{_html_styles()} + + +
+ +{_html_script(nodes_json, edges_json, legend_json)} +{_hyperedge_script(hyperedges_json)} + +""" + + Path(output_path).write_text(html, encoding="utf-8") # nosec diff --git a/graphify/extract.py b/graphify/extract.py index be7e99448..9de0c7622 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -7,6 +7,7 @@ import os import re import sys +from collections import Counter from dataclasses import dataclass, field from pathlib import Path from typing import Any, Callable @@ -20,6 +21,7 @@ run_language_resolvers, ) from .ruby_resolution import resolve_ruby_member_calls +from .pascal_resolution import resolve_pascal_inherited_calls # --- migrated to graphify/extractors/ (see graphify/extractors/MIGRATION.md) --- from graphify.extractors.base import ( # noqa: F401 @@ -28,17 +30,121 @@ _make_id, _read_text, ) +from graphify.extractors.apex import extract_apex # noqa: F401 +from graphify.extractors.bash import extract_bash # noqa: F401 from graphify.extractors.blade import extract_blade # noqa: F401 from graphify.extractors.csharp import ( + CsharpNameResolver, _resolve_cross_file_csharp_imports, _resolve_csharp_type_references, ) +from graphify.extractors.dart import extract_dart # noqa: F401 +from graphify.extractors.dm import extract_dm, extract_dmf, extract_dmi, extract_dmm # noqa: F401 from graphify.extractors.elixir import extract_elixir # noqa: F401 +from graphify.extractors.fortran import _cpp_preprocess, extract_fortran # noqa: F401 +from graphify.extractors.go import extract_go # noqa: F401 +from graphify.extractors.json_config import extract_json # noqa: F401 +from graphify.extractors.markdown import extract_markdown # noqa: F401 +from graphify.extractors.pascal_forms import extract_delphi_form, extract_lazarus_form # noqa: F401 +from graphify.extractors.powershell import extract_powershell, extract_powershell_manifest # noqa: F401 from graphify.extractors.razor import extract_razor # noqa: F401 +from graphify.extractors.rust import extract_rust # noqa: F401 +from graphify.extractors.sln import extract_sln # noqa: F401 +from graphify.extractors.sql import extract_sql # noqa: F401 +from graphify.extractors.terraform import extract_terraform # noqa: F401 +from graphify.extractors.verilog import extract_verilog # noqa: F401 from graphify.extractors.zig import extract_zig # noqa: F401 from graphify.security import sanitize_metadata from graphify.paths import disambiguate_ambiguous_candidates +from graphify.extractors.models import LanguageConfig, _JS_CACHE_BYPASS_SUFFIXES, _NamespaceExportFact, _StarExportFact, _SymbolAliasFact, _SymbolDeclarationFact, _SymbolExportFact, _SymbolImportFact, _SymbolResolutionFacts, _SymbolUseFact, _WORKSPACE_PACKAGE_CACHE # noqa: E402,F401 + +from graphify.extractors.resolution import ( # noqa: E402,F401 + _DECLDEF_HEADER_SUFFIXES, + _DECLDEF_IMPL_SUFFIXES, + _EXPORT_CONDITION_PRIORITY, + _JS_INDEX_FILES, + _JS_PRIMITIVE_TYPES, + _JS_RESOLVE_EXTS, + _TSCONFIG_ALIAS_CACHE, + _VUE_SCRIPT_LANG_RE, + _VUE_SCRIPT_RE, + _WORKSPACE_MANIFEST_NAMES, + _apply_symbol_resolution_facts, + _augment_symbol_resolution_edges, + _collect_js_symbol_resolution_facts, + _collect_python_symbol_resolution_facts, + _contained_in_package, + _decldef_class_stem, + _disambiguate_colliding_node_ids, + _find_workspace_root, + _is_type_like_definition, + _js_call_identifier, + _js_default_export_name, + _js_default_import_name, + _js_export_clause, + _js_export_statement_is_star, + _js_exported_declaration_names, + _js_lexical_aliases, + _js_module_specifier, + _js_named_specifiers, + _js_namespace_export_name, + _js_source_path, + _js_top_level_function_bodies, + _load_tsconfig_aliases, + _load_tsconfig_base_url, + _load_workspace_packages, + _match_tsconfig_alias, + _merge_decl_def_classes, + _node_disambiguation_source_key, + _package_entry_candidates, + _parse_js_tree, + _parse_python_tree, + _pascal_class_stem_cache, + _pascal_project_root, + _pascal_resolve_class, + _pascal_resolve_unit, + _pascal_unit_cache, + _pnpm_workspace_globs, + _python_call_identifier, + _python_import_from_module, + _python_imported_names, + _python_top_level_function_bodies, + _read_tsconfig_aliases, + _resolve_c_include_path, + _resolve_cross_file_imports, + _resolve_cross_file_java_imports, + _resolve_export_target, + _resolve_java_type_references, + _resolve_php_type_references, + _resolve_js_import_path, + _resolve_js_import_target, + _resolve_js_module_path, + _resolve_lua_import_target, + _resolve_python_module_path, + _resolve_tsconfig_alias, + _resolve_workspace_import, + _source_key, + _strip_jsonc, + _ts_collect_type_refs, + _ts_heritage_clause_entries, + _ts_walk_class_members, + _vue_mask_non_script, + _walk_js_tree, + _walk_python_tree, + _workspace_globs, +) + +from graphify.symbol_resolution import resolve_bash_source_edges # noqa: E402 + +from graphify.extractors.engine import REFERENCE_CONTEXTS, _CSHARP_TYPE_PARAMETER_SCOPE_DECLARATIONS, _C_PRIMITIVE_TYPE_NODES, _JAVA_BUILTIN_TYPES, _JAVA_TYPE_PARAMETER_SCOPE_DECLARATIONS, _JS_FUNCTION_VALUE_TYPES, _JS_SCOPE_BOUNDARY, _PYTHON_ANNOTATION_NOISE, _PYTHON_TYPE_CONTAINERS, _RUBY_CLASS_FACTORIES, _c_collect_type_refs, _cpp_collect_type_refs, _cpp_declarator_name, _cpp_local_var_types, _csharp_attribute_names, _csharp_classify_base, _csharp_collect_type_refs, _csharp_extra_walk, _csharp_member_type_table, _csharp_namespace_id, _csharp_namespace_name, _csharp_pre_scan_interfaces, _csharp_type_parameters_in_scope, _dynamic_import_js, _extract_generic, _find_body, _find_require_call, _get_cpp_func_name, _java_annotation_names, _java_collect_type_refs, _java_extra_walk, _java_type_parameters_in_scope, _js_collect_pattern_idents, _js_dispatch_value_idents, _js_extra_walk, _js_local_bound_names, _js_member_assignment_target, _js_module_bound_names, _kotlin_collect_type_refs, _kotlin_function_return_type_node, _kotlin_property_type_node, _kotlin_user_type_name, _php_collect_type_refs, _php_method_return_type_node, _php_name_text, _python_collect_assignment_targets, _python_collect_param_refs, _python_collect_type_refs, _python_local_bound_names, _python_module_bound_names, _python_param_names, _read_csharp_type_name, _require_imports_js, _ruby_const_last_name, _ruby_extra_walk, _ruby_local_class_bindings, _ruby_new_class_name, _scala_collect_type_refs, _semantic_reference_edge, _source_location, _swift_classify_base, _swift_collect_type_refs, _swift_constructor_type, _swift_declaration_keyword, _swift_extra_walk, _swift_local_var_types, _swift_pre_scan, _swift_property_name, _swift_property_type_node, _swift_receiver_name, _swift_user_type_name, _ts_decorator_name, _ts_descendant_decorators, _ts_emit_decorator_edges, _ts_extra_walk, _ts_method_name, _ts_receiver_type_table # noqa: E402,F401 + +from graphify.extractors.pascal import _PAS_BEGIN_END_TOKEN_RE, _PAS_CALL_RE, _PAS_END_SEMI_RE, _PAS_IMPL_HEADER_RE, _PAS_KEYWORDS, _PAS_METHOD_DECL_RE, _PAS_MODULE_RE, _PAS_TOKEN_RE, _PAS_TYPE_HEADER_RE, _PAS_USES_RE, _extract_pascal_regex, _pascal_find_body, _pascal_split_bases, _pascal_split_sections, _pascal_split_uses, _pascal_strip_comments, extract_pascal # noqa: E402,F401 + +from graphify.extractors.objc import _objc_local_var_types, extract_objc # noqa: E402,F401 + +from graphify.extractors.julia import extract_julia # noqa: E402,F401 + _RECURSION_LIMIT = 10_000 # Language built-in globals that AST may classify as call targets when used as @@ -67,10 +173,6 @@ def _safe_extract(extractor: Callable, path: Path) -> dict: return {"nodes": [], "edges": [], "error": f"{type(e).__name__}: {e}"} - - - - def _file_node_id(rel_path: Path) -> str: """File-level node ID matching the skill.md spec: ``{parent_dir}_{stem}`` — one parent directory level, no extension. ``rel_path`` MUST be relative to @@ -81,17 +183,76 @@ def _file_node_id(rel_path: Path) -> str: return _make_id(_file_stem(rel_path)) -def _csharp_namespace_id(dotted_name: str) -> str: - digest = hashlib.sha1(dotted_name.encode("utf-8")).hexdigest()[:16] - return f"csharp_namespace:{digest}" - - -_TSCONFIG_ALIAS_CACHE: dict[str, dict[str, list[str]]] = {} -_WORKSPACE_PACKAGE_CACHE: dict[str, dict[str, Path]] = {} -_WORKSPACE_MANIFEST_NAMES = ("pnpm-workspace.yaml", "package.json") -_JS_CACHE_BYPASS_SUFFIXES = {".js", ".jsx", ".mjs", ".ts", ".tsx", ".mts", ".cts", ".vue", ".svelte"} -_JS_RESOLVE_EXTS = (".ts", ".tsx", ".mts", ".cts", ".svelte", ".js", ".jsx", ".mjs") -_JS_INDEX_FILES = ("index.ts", "index.tsx", "index.svelte", "index.js", "index.jsx", "index.mjs") +def _repoint_python_package_imports(paths, all_nodes, all_edges, root) -> None: + """Repoint Python absolute-import edges to the real file node under a nested + (e.g. ``src/``) package root (#2072). + + Absolute imports target an id derived from the dotted module path + (``_make_id('pkg.mod')`` -> ``pkg_mod``), but file-node ids are + scan-root-relative (``src_pkg_mod`` when the code lives under ``src/``), so + the edge dangles and is silently dropped — the graph loses most ``imports`` + edges purely because of where the scan started. Build an alias map from the + dotted-module id to the real file-node id by detecting each ``.py`` file's + package root (the contiguous run of ancestor dirs carrying ``__init__.py``) + and rewrite matching ``imports``/``imports_from`` edge targets. Guards: never + shadow an existing node id, and drop an alias claimed by more than one file + (ambiguous -> leave dangling, as before). Files whose package root IS the + scan root are skipped (ids already coincide).""" + try: + root = Path(root).resolve() + except OSError: + root = Path(root) + node_ids = {n.get("id") for n in all_nodes if isinstance(n, dict)} + alias_to_files: dict[str, set[str]] = {} + for p in paths: + if p.suffix.lower() not in (".py", ".pyi"): + continue + try: + rel = Path(p).resolve().relative_to(root) + except (ValueError, OSError): + continue + parts = rel.parts + if len(parts) < 2: + continue # top-level file: scan-root-relative id already matches + d = Path(p).resolve().parent + levels = 0 + # Bounded by the number of dirs between the file and the scan root, so a + # pathological `/__init__.py` chain can't loop forever. + while levels < len(parts) - 1 and (d / "__init__.py").is_file(): + levels += 1 + d = d.parent + if levels == 0: + continue # not inside a package (namespace pkg / loose module) + mod_parts = parts[-(levels + 1):] # package dirs + the file itself + if len(mod_parts) == len(parts): + continue # package root == scan root: file-node id already coincides + file_node = _file_node_id(rel) + alias = _make_id(str(Path(*mod_parts).with_suffix(""))) + alias_to_files.setdefault(alias, set()).add(file_node) + if p.name in ("__init__.py", "__init__.pyi") and len(mod_parts) > 1: + # `import pkg` / `from pkg import x` targets the package-dir id. + pkg_alias = _make_id(str(Path(*mod_parts[:-1]))) + alias_to_files.setdefault(pkg_alias, set()).add(file_node) + alias_map = { + a: next(iter(fs)) + for a, fs in alias_to_files.items() + if len(fs) == 1 and a not in node_ids + } + if not alias_map: + return + for e in all_edges: + # Only repoint edges emitted from a Python file: a non-Python import edge + # (e.g. C# `using Pkg.Mod;`, Java/Go dotted imports) can have a dangling + # target string that coincides with a Python alias, and repointing it + # would fabricate a cross-language import edge (#2072 review). + if ( + isinstance(e, dict) + and e.get("relation") in ("imports", "imports_from") + and str(e.get("source_file", "")).lower().endswith((".py", ".pyi")) + ): + tgt = e.get("target") + if tgt in alias_map: + e["target"] = alias_map[tgt] SEMANTIC_RELATIONS = frozenset({ @@ -99,773 +260,21 @@ def _csharp_namespace_id(dotted_name: str) -> str: "calls", "imports", "imports_from", "re_exports", "contains", "method", }) -REFERENCE_CONTEXTS = frozenset({ - "field", "parameter_type", "return_type", "generic_arg", "attribute", "value", "type", -}) - - -def _source_location(line: int | str | None) -> str | None: - if line is None: - return None - if isinstance(line, str): - return line if line.startswith("L") else f"L{line}" - return f"L{line}" - - -def _semantic_reference_edge( - source: str, - target: str, - context: str, - source_file: str, - line: int | str | None, -) -> dict: - if context not in REFERENCE_CONTEXTS: - raise ValueError(f"unknown reference context: {context}") - return { - "source": source, - "target": target, - "relation": "references", - "context": context, - "confidence": "EXTRACTED", - "source_file": source_file, - "source_location": _source_location(line), - "weight": 1.0, - } - - -def _resolve_js_import_path(candidate: Path) -> Path: - """Resolve a JS/TS/Svelte import target to a local file when it exists.""" - candidate = Path(os.path.normpath(candidate)) - if candidate.is_file(): - return candidate - - # TS ESM convention: imports often spell .js/.jsx while source is .ts/.tsx. - if candidate.suffix == ".js": - ts_candidate = candidate.with_suffix(".ts") - if ts_candidate.is_file(): - return ts_candidate - elif candidate.suffix == ".jsx": - tsx_candidate = candidate.with_suffix(".tsx") - if tsx_candidate.is_file(): - return tsx_candidate - - # Append extensions to the full filename, which covers extensionless imports, - # multi-dot helpers, and Svelte 5 rune files like Foo.svelte.ts. - for ext in _JS_RESOLVE_EXTS: - with_ext = candidate.parent / f"{candidate.name}{ext}" - if with_ext.is_file(): - return with_ext - - # Only fall back to directory indexes after file candidates lose. - if candidate.is_dir(): - for index_name in _JS_INDEX_FILES: - index_candidate = candidate / index_name - if index_candidate.is_file(): - return index_candidate - - return candidate - - -def _strip_jsonc(text: str) -> str: - """Strip // line comments, /* */ block comments, and trailing commas from JSONC. - - Preserves string contents (including // and /* inside strings) by skipping over - quoted spans first. Required for tsconfig.json files generated by SvelteKit, - NestJS, Vite, T3, Astro, etc., which use JSONC by default (#700). - """ - # Remove block and line comments while leaving string literals untouched. - pattern = re.compile( - r'"(?:\\.|[^"\\])*"' # double-quoted string (with escapes) - r"|/\*.*?\*/" # /* block comment */ - r"|//[^\n]*", # // line comment - re.DOTALL, - ) - - def _replace(match: re.Match) -> str: - token = match.group(0) - if token.startswith('"'): - return token - return "" - - stripped = pattern.sub(_replace, text) - # Remove trailing commas before } or ] (allowing whitespace between). - stripped = re.sub(r",(\s*[}\]])", r"\1", stripped) - return stripped - - -def _read_tsconfig_aliases(tsconfig: Path, base_dir: Path, seen: set) -> dict[str, list[str]]: - """Recursively read path aliases from a tsconfig, following extends chains. - - Child config paths override parent. Circular extends are detected via seen set. - npm package configs (e.g. @tsconfig/svelte) are skipped since they're not on disk. - Handles JSONC (comments + trailing commas) which is the default tsconfig format - for SvelteKit, NestJS, Vite, T3, Astro, etc. (#700). - """ - if str(tsconfig) in seen: - return {} - seen.add(str(tsconfig)) - try: - raw = tsconfig.read_text(encoding="utf-8") - except Exception as e: - print(f" warning: could not read {tsconfig} ({type(e).__name__}: {e})", file=sys.stderr, flush=True) - return {} - try: - data = json.loads(raw) - except json.JSONDecodeError: - try: - data = json.loads(_strip_jsonc(raw)) - except json.JSONDecodeError as e: - print(f" warning: failed to parse {tsconfig} as JSON/JSONC ({e.msg} at line {e.lineno} col {e.colno})", file=sys.stderr, flush=True) - return {} - except Exception as e: - print(f" warning: failed to parse {tsconfig} ({type(e).__name__}: {e})", file=sys.stderr, flush=True) - return {} - - aliases: dict[str, list[str]] = {} - # `extends` may be a string or, since TypeScript 5.0, an array of paths. - # For an array, parents are processed in order with later entries - # overriding earlier ones; the extending config (paths below) overrides - # all parents. Without the list branch, an array `extends` raised - # `AttributeError: 'list' object has no attribute 'startswith'`, which - # _safe_extract turned into a skip of the whole file. - extends = data.get("extends") - if isinstance(extends, str): - extends_list = [extends] - elif isinstance(extends, list): - extends_list = [e for e in extends if isinstance(e, str)] - else: - extends_list = [] - for ext in extends_list: - # Skip scoped npm package configs (e.g. @tsconfig/svelte) — not on disk. - if not ext or ext.startswith("@"): - continue - extended_path = (base_dir / ext).resolve() - if not extended_path.suffix: - extended_path = extended_path.with_suffix(".json") - if extended_path.exists(): - aliases.update(_read_tsconfig_aliases(extended_path, extended_path.parent, seen)) - - # tsconfig `paths` are resolved relative to `baseUrl` (itself relative to - # the tsconfig's directory), not the tsconfig directory directly. Honoring - # baseUrl is required for the common monorepo / NestJS layout where - # baseUrl points at a subdirectory, e.g. baseUrl "./src" with - # "@services/*": ["services/*"] must resolve to /src/services rather - # than /services. Defaults to "." so configs without baseUrl (paths - # relative to the tsconfig dir, the TS 4.1+ behavior) keep working. - compiler_options = data.get("compilerOptions", {}) - base_url = compiler_options.get("baseUrl") or "." - paths_base = base_dir / base_url - paths = compiler_options.get("paths", {}) - for alias, targets in paths.items(): - if not targets: - continue - # Keep ALL targets in declared order — tsc tries each until one resolves - # on disk. Discarding the fallbacks (#1531) misresolved/dropped imports - # whose file lived at a non-first target. Preserve wildcard tokens in - # both sides until the resolver substitutes the captured segment, then - # normalizes the concrete path (#927). Empty/non-string entries are skipped. - target_patterns = [ - str(paths_base / t) - for t in targets - if isinstance(t, str) and t - ] - if target_patterns: - aliases[alias] = target_patterns - - return aliases - - -def _load_tsconfig_aliases(start_dir: Path) -> dict[str, list[str]]: - """Walk up from start_dir to find tsconfig.json and return compilerOptions.paths aliases. - - Follows extends chains so SvelteKit/Nuxt/NestJS inherited aliases are included. - Returns a dict mapping alias patterns to ordered resolved target patterns; - wildcard tokens remain intact for substitution during resolution (#927). - Result is cached by tsconfig path string. - """ - current = start_dir.resolve() - for candidate in [current, *current.parents]: - tsconfig = candidate / "tsconfig.json" - if tsconfig.exists(): - key = str(tsconfig) - if key not in _TSCONFIG_ALIAS_CACHE: - _TSCONFIG_ALIAS_CACHE[key] = _read_tsconfig_aliases(tsconfig, candidate, seen=set()) - return _TSCONFIG_ALIAS_CACHE[key] - return {} - - -def _match_tsconfig_alias(raw: str, pattern: str) -> "tuple[tuple[int, int], str, bool] | None": - """Return (specificity, captured text, is_wildcard) when pattern matches raw. - - Exact aliases win first. Wildcard aliases follow TypeScript's longest-prefix - rule. The final branch preserves Graphify's existing support for treating a - non-wildcard alias as a directory prefix, but only after real wildcard matches. - """ - if "*" in pattern: - if pattern.count("*") != 1: - return None - prefix, suffix = pattern.split("*", 1) - if not raw.startswith(prefix) or not raw.endswith(suffix): - return None - end = len(raw) - len(suffix) if suffix else len(raw) - if end < len(prefix): - return None - return (1, -len(prefix)), raw[len(prefix):end], True - - if raw == pattern: - return (0, -len(pattern)), "", False - - prefix = pattern.rstrip("/") - if prefix and raw.startswith(prefix + "/"): - return (2, -len(prefix)), raw[len(prefix):].lstrip("/"), False - return None - - -def _resolve_tsconfig_alias(raw: str, aliases: dict[str, list[str]]) -> "Path | None": - """Resolve `raw` against the most specific matching tsconfig alias pattern. - - Within that pattern, try targets in declared order and return the first whose - candidate resolves to a real file. If none exist, return the first candidate - so existing phantom/external-edge behavior stays unchanged. - """ - best: "tuple[tuple[int, int], str, bool, list[str]] | None" = None - for pattern, targets in aliases.items(): - match = _match_tsconfig_alias(raw, pattern) - if match is None: - continue - specificity, captured, is_wildcard = match - if best is None or specificity < best[0]: - best = specificity, captured, is_wildcard, targets - - if best is None: - return None - - _, captured, is_wildcard, targets = best - first = None - for target in targets: - if is_wildcard: - # TypeScript substitutes only when the matched star is non-empty. - substituted = target.replace("*", captured, 1) if captured else target - cand = Path(os.path.normpath(substituted)) - else: - cand = Path(target) - if captured: - cand = Path(os.path.normpath(cand / captured)) - resolved = _resolve_js_import_path(cand) - if resolved.is_file(): - return resolved - if first is None: - first = cand - return first - - -def _find_workspace_root(start_dir: Path) -> Path | None: - current = start_dir.resolve() - for candidate in [current, *current.parents]: - if (candidate / "pnpm-workspace.yaml").exists(): - return candidate - package_json = candidate / "package.json" - if package_json.is_file(): - try: - data = json.loads(package_json.read_text(encoding="utf-8")) - except Exception: - continue - if "workspaces" in data: - return candidate - return None - - -def _pnpm_workspace_globs(workspace_file: Path) -> list[str]: - globs: list[str] = [] - in_packages = False - for raw_line in workspace_file.read_text(encoding="utf-8", errors="replace").splitlines(): - line = raw_line.strip() - if not line or line.startswith("#"): - continue - if line.startswith("packages:"): - in_packages = True - continue - if in_packages and line.startswith("-"): - value = line[1:].strip().strip("'\"") - if value and not value.startswith("!"): - globs.append(value) - continue - if in_packages and not raw_line.startswith((" ", "\t")): - break - return globs - - -def _workspace_globs(root: Path) -> list[str]: - pnpm_workspace = root / "pnpm-workspace.yaml" - if pnpm_workspace.exists(): - return _pnpm_workspace_globs(pnpm_workspace) - - package_json = root / "package.json" - try: - data = json.loads(package_json.read_text(encoding="utf-8")) - except Exception: - return [] - - workspaces = data.get("workspaces") - if isinstance(workspaces, list): - return [item for item in workspaces if isinstance(item, str) and not item.startswith("!")] - if isinstance(workspaces, dict): - packages = workspaces.get("packages") - if isinstance(packages, list): - return [item for item in packages if isinstance(item, str) and not item.startswith("!")] - return [] - - -def _load_workspace_packages(start_dir: Path) -> dict[str, Path]: - root = _find_workspace_root(start_dir) - if root is None: - return {} - manifest_mtimes = tuple( - (name, (root / name).stat().st_mtime_ns) - for name in _WORKSPACE_MANIFEST_NAMES - if (root / name).is_file() - ) - key = str((root, manifest_mtimes)) - if key in _WORKSPACE_PACKAGE_CACHE: - return _WORKSPACE_PACKAGE_CACHE[key] - - packages: dict[str, Path] = {} - for pattern in _workspace_globs(root): - package_dirs: list[Path] = [root] if pattern in (".", "./") else list(root.glob(pattern)) - for package_dir in package_dirs: - manifest = package_dir / "package.json" - if not manifest.is_file(): - continue - try: - data = json.loads(manifest.read_text(encoding="utf-8")) - except Exception: - continue - name = data.get("name") - if isinstance(name, str) and name: - packages[name] = package_dir - _WORKSPACE_PACKAGE_CACHE[key] = packages - return packages - # Condition keys consulted when resolving an `exports` target, in priority # order. `default` is Node's catch-all and must be consulted LAST so a more # specific condition (source/import/module/etc.) wins when several match. -_EXPORT_CONDITION_PRIORITY = ( - "source", "import", "module", "svelte", "types", "require", "default", -) - - -def _resolve_export_target(value: Any) -> str | None: - """Resolve an `exports` map value (string or condition object) to a - relative target string, honouring _EXPORT_CONDITION_PRIORITY for objects - and recursing into nested condition objects.""" - if isinstance(value, str): - return value - if isinstance(value, dict): - for cond in _EXPORT_CONDITION_PRIORITY: - v = value.get(cond) - if isinstance(v, str): - return v - if isinstance(v, dict): - nested = _resolve_export_target(v) - if nested: - return nested - return None - - -def _contained_in_package(resolved: Path, package_dir: Path) -> bool: - """Guard against `exports` targets that escape the package directory - (e.g. "./evil": "../../../etc/passwd"). Only accept paths that stay - within package_dir after resolution.""" - try: - return resolved.resolve().is_relative_to(package_dir.resolve()) - except ValueError: - return False - - -def _package_entry_candidates(package_dir: Path, subpath: str) -> list[Path]: - manifest = package_dir / "package.json" - manifest_data: dict[str, Any] = {} - try: - manifest_data = json.loads(manifest.read_text(encoding="utf-8")) - except Exception: - pass - - if subpath: - # Consult the package's `exports` subpath map before the bare-path - # fallback (#1308): "./browser" -> conditions -> file, plus single - # wildcard "./*" patterns. Targets that escape the package dir are - # rejected; resolution then falls through to the bare path. - exports = manifest_data.get("exports") - if isinstance(exports, dict): - subpath_key = "./" + subpath - target = _resolve_export_target(exports.get(subpath_key)) - if target: - candidate = package_dir / target - if _contained_in_package(candidate, package_dir): - return [candidate] - else: - for pattern, pattern_value in exports.items(): - if "*" in pattern and pattern.count("*") == 1: - prefix, suffix = pattern.split("*", 1) - if (subpath_key.startswith(prefix) - and (not suffix or subpath_key.endswith(suffix))): - matched = subpath_key[len(prefix):len(subpath_key) - len(suffix) if suffix else None] - resolved = _resolve_export_target(pattern_value) - if resolved and "*" in resolved: - candidate = package_dir / resolved.replace("*", matched) - if _contained_in_package(candidate, package_dir): - return [candidate] - return [package_dir / subpath] - - exports = manifest_data.get("exports") - if isinstance(exports, str): - return [package_dir / exports] - if isinstance(exports, dict): - dot_target = _resolve_export_target(exports.get(".")) - if dot_target: - return [package_dir / dot_target] - - candidates: list[Path] = [] - for key in ("svelte", "module", "main", "types"): - value = manifest_data.get(key) - if isinstance(value, str): - candidates.append(package_dir / value) - candidates.append(package_dir / "src/index") - candidates.append(package_dir / "index") - return candidates - - -def _resolve_workspace_import(raw: str, start_dir: Path) -> Path | None: - packages = _load_workspace_packages(start_dir) - for package_name, package_dir in packages.items(): - if raw == package_name: - subpath = "" - elif raw.startswith(package_name + "/"): - subpath = raw[len(package_name) + 1:] - else: - continue - for candidate in _package_entry_candidates(package_dir, subpath): - resolved = _resolve_js_import_path(candidate) - if resolved.is_file(): - return resolved - return None - - -def _resolve_js_module_path(raw: str | Path, start_dir: Path | None = None) -> Path | None: - """Resolve a JS/TS module path or specifier to a local source file. - - With a Path argument this preserves the path-based helper API used by - import-extension tests. With a string plus start_dir it resolves JS/TS - module specifiers including relative paths, tsconfig aliases, and workspace - packages. - """ - if isinstance(raw, Path): - return _resolve_js_import_path(raw) - if start_dir is None: - return _resolve_js_import_path(Path(raw)) - if raw.startswith("."): - return _resolve_js_import_path(start_dir / raw) - - aliases = _load_tsconfig_aliases(start_dir) - hit = _resolve_tsconfig_alias(raw, aliases) - if hit is not None: - return _resolve_js_import_path(hit) - - return _resolve_workspace_import(raw, start_dir) # ── LanguageConfig dataclass ───────────────────────────────────────────────── -@dataclass -class LanguageConfig: - ts_module: str # e.g. "tree_sitter_python" - ts_language_fn: str = "language" # attr to call: e.g. tslang.language() - - class_types: frozenset = frozenset() - function_types: frozenset = frozenset() - import_types: frozenset = frozenset() - call_types: frozenset = frozenset() - static_prop_types: frozenset = frozenset() - helper_fn_names: frozenset = frozenset() - container_bind_methods: frozenset = frozenset() - event_listener_properties: frozenset = frozenset() - - # Name extraction - name_field: str = "name" - name_fallback_child_types: tuple = () - - # Body detection - body_field: str = "body" - body_fallback_child_types: tuple = () # e.g. ("declaration_list", "compound_statement") - - # Call name extraction - call_function_field: str = "function" # field on call node for callee - call_accessor_node_types: frozenset = frozenset() # member/attribute nodes - call_accessor_field: str = "attribute" # field on accessor for method name - call_accessor_object_field: str = "" # field on accessor for the receiver/object - - # Stop recursion at these types in walk_calls - function_boundary_types: frozenset = frozenset() - - # Import handler: called for import nodes instead of generic handling - import_handler: Callable | None = None - - # Optional custom name resolver for functions (C, C++ declarator unwrapping) - resolve_function_name_fn: Callable | None = None - - # Extra label formatting for functions: if True, functions get "name()" label - function_label_parens: bool = True - - # Extra walk hook called after generic dispatch (for JS arrow functions, C# namespaces, etc.) - extra_walk_fn: Callable | None = None - # ── Generic helpers ─────────────────────────────────────────────────────────── - -_PYTHON_TYPE_CONTAINERS = frozenset({ - "list", "dict", "set", "tuple", "frozenset", "type", - "List", "Dict", "Set", "Tuple", "FrozenSet", "Type", - "Optional", "Union", "Sequence", "Iterable", "Mapping", "MutableMapping", - "Iterator", "Callable", "Awaitable", "AsyncIterable", "AsyncIterator", "Coroutine", - "Generator", "AsyncGenerator", "ContextManager", "AsyncContextManager", - "Annotated", "ClassVar", "Final", "Literal", "Concatenate", "ParamSpec", "TypeVar", - "None", "Ellipsis", -}) - # Scalar builtins and test-mock names that appear as type annotations but carry # no useful semantic meaning as graph nodes (#1147). Suppressed at the annotation # walker level so they are never created as nodes or emitted as edges. -_PYTHON_ANNOTATION_NOISE = frozenset({ - # scalar builtins - "str", "int", "float", "bool", "bytes", "bytearray", "complex", "object", - "True", "False", - # unittest.mock - "MagicMock", "Mock", "AsyncMock", "NonCallableMock", - "NonCallableMagicMock", "PropertyMock", "patch", "sentinel", -}) - - -def _python_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None: - """Walk a Python type annotation; append (name, role) where role is 'type' or 'generic_arg'. - - Builtin/typing containers (list, dict, Optional, Union, …) are not emitted as refs themselves, - but their nested type arguments still count as generic_arg. - """ - if node is None: - return - t = node.type - if t == "type": - for c in node.children: - if c.is_named: - _python_collect_type_refs(c, source, generic, out) - return - if t == "identifier": - name = _read_text(node, source) - if name and name not in _PYTHON_TYPE_CONTAINERS and name not in _PYTHON_ANNOTATION_NOISE: - out.append((name, "generic_arg" if generic else "type")) - return - if t == "attribute": - tail = _read_text(node, source).rsplit(".", 1)[-1] - if tail and tail not in _PYTHON_TYPE_CONTAINERS and tail not in _PYTHON_ANNOTATION_NOISE: - out.append((tail, "generic_arg" if generic else "type")) - return - if t == "generic_type": - for c in node.children: - if c.type == "identifier": - container = _read_text(c, source) - if container and container not in _PYTHON_TYPE_CONTAINERS and container not in _PYTHON_ANNOTATION_NOISE: - out.append((container, "generic_arg" if generic else "type")) - elif c.type == "type_parameter": - for sub in c.children: - if sub.is_named: - _python_collect_type_refs(sub, source, True, out) - return - if t == "subscript": - value = node.child_by_field_name("value") - if value is not None: - _python_collect_type_refs(value, source, generic, out) - for c in node.children: - if c is value or not c.is_named: - continue - _python_collect_type_refs(c, source, True, out) - return - if node.is_named: - for c in node.children: - if c.is_named: - _python_collect_type_refs(c, source, generic, out) - - -def _csharp_pre_scan_interfaces(root_node, source: bytes) -> set[str]: - """Return names declared as `interface` in this C# compilation unit.""" - out: set[str] = set() - stack = [root_node] - while stack: - n = stack.pop() - if n.type == "interface_declaration": - name_node = n.child_by_field_name("name") - if name_node is not None: - text = _read_text(name_node, source) - if text: - out.add(text) - stack.extend(n.children) - return out - - -def _csharp_classify_base(name: str, interface_names: set[str]) -> str: - """`implements` if the base name is an interface (declared or by I-prefix convention), else `inherits`.""" - if name in interface_names: - return "implements" - if len(name) >= 2 and name[0] == "I" and name[1].isupper(): - return "implements" - return "inherits" - - -_CSHARP_TYPE_PARAMETER_SCOPE_DECLARATIONS = frozenset({ - "class_declaration", - "interface_declaration", - "record_declaration", - "struct_declaration", - "method_declaration", -}) - - -def _csharp_type_parameters_in_scope(node, source: bytes) -> frozenset[str]: - """Return C# type-parameter names visible from ``node``.""" - names: set[str] = set() - scope = node - while scope is not None: - if scope.type in _CSHARP_TYPE_PARAMETER_SCOPE_DECLARATIONS: - for child in scope.children: - if child.type != "type_parameter_list": - continue - for param in child.children: - if param.type == "type_parameter": - name_node = next( - (sub for sub in param.children if sub.type == "identifier"), - None, - ) - if name_node is not None: - name = _read_text(name_node, source) - if name: - names.add(name) - elif param.type == "identifier": - name = _read_text(param, source) - if name: - names.add(name) - scope = scope.parent - return frozenset(names) - - -def _csharp_collect_type_refs( - node, - source: bytes, - generic: bool, - out: list[tuple[str, str, bool, str]], - skip: frozenset[str] | None = None, -) -> None: - """Walk a C# type expression; append (name, role, qualified, qualifier) tuples.""" - if node is None: - return - if skip is None: - skip = _csharp_type_parameters_in_scope(node, source) - t = node.type - if t == "predefined_type": - return - if t == "identifier": - name = _read_text(node, source) - if name and name not in skip: - out.append((name, "generic_arg" if generic else "type", False, "")) - return - if t == "qualified_name": - prefix, _, text = _read_text(node, source).rpartition(".") - text = text.split("<", 1)[0] - if text and text not in skip: - out.append((text, "generic_arg" if generic else "type", True, prefix)) - return - if t == "generic_name": - name_child = node.child_by_field_name("name") - if name_child is None: - for sub in node.children: - if sub.type == "identifier": - name_child = sub - break - if name_child is not None: - qualified = name_child.type == "qualified_name" - prefix, _, name = _read_text(name_child, source).rpartition(".") - if name and name not in skip: - out.append((name, "generic_arg" if generic else "type", qualified, prefix if qualified else "")) - for sub in node.children: - if sub.type == "type_argument_list": - for arg in sub.children: - if arg.is_named: - _csharp_collect_type_refs(arg, source, True, out, skip) - return - if t in ("nullable_type", "array_type", "pointer_type", "ref_type"): - for c in node.children: - if c.is_named: - _csharp_collect_type_refs(c, source, generic, out, skip) - return - if node.is_named: - for c in node.children: - if c.is_named: - _csharp_collect_type_refs(c, source, generic, out, skip) - - -def _csharp_attribute_names(method_node, source: bytes) -> list[tuple[str, bool, str]]: - """Collect attribute names from a C# method/declaration's attribute_list children.""" - names: list[tuple[str, bool, str]] = [] - skip = _csharp_type_parameters_in_scope(method_node, source) - for child in method_node.children: - if child.type != "attribute_list": - continue - for attr in child.children: - if attr.type != "attribute": - continue - name_node = attr.child_by_field_name("name") - if name_node is None: - for sub in attr.children: - if sub.type in ("identifier", "qualified_name"): - name_node = sub - break - if name_node is not None: - qualified = name_node.type == "qualified_name" - prefix, _, text = _read_text(name_node, source).rpartition(".") - if text and text not in skip: - names.append((text, qualified, prefix if qualified else "")) - return names - - -_JAVA_TYPE_PARAMETER_SCOPE_DECLARATIONS = frozenset({ - "class_declaration", - "interface_declaration", - "record_declaration", - "method_declaration", - "constructor_declaration", -}) - - -def _java_type_parameters_in_scope(node, source: bytes) -> frozenset[str]: - """Return Java type-parameter names visible from ``node``.""" - names: set[str] = set() - scope = node - while scope is not None: - if scope.type in _JAVA_TYPE_PARAMETER_SCOPE_DECLARATIONS: - params = scope.child_by_field_name("type_parameters") - if params is not None: - for param in params.children: - if param.type != "type_parameter": - continue - name_node = next( - (child for child in param.children if child.type == "type_identifier"), - None, - ) - if name_node is not None: - names.add(_read_text(name_node, source)) - scope = scope.parent - return frozenset(names) # java.lang (auto-imported) plus the ubiquitous java.util / java.io / java.time / @@ -876,917 +285,40 @@ def _java_type_parameters_in_scope(node, source: bytes) -> frozenset[str]: # type-ref walker so they are never created as nodes or emitted as edges. The # boxed-scalar/`void` primitives are already dropped by grammar node type above; # these are the class/interface names the grammar reports as identifiers. -_JAVA_BUILTIN_TYPES = frozenset({ - # java.lang — core - "Object", "String", "CharSequence", "StringBuilder", "StringBuffer", - "Number", "Byte", "Short", "Integer", "Long", "Float", "Double", - "Boolean", "Character", "Void", "Class", "Enum", "Record", "Math", - "System", "Thread", "Runnable", "Comparable", "Iterable", "Cloneable", - "AutoCloseable", "Appendable", "Readable", "Process", "ProcessBuilder", - "Runtime", "Package", "ThreadLocal", "InheritableThreadLocal", - # java.lang — throwables - "Throwable", "Exception", "RuntimeException", "Error", - "IllegalArgumentException", "IllegalStateException", "NullPointerException", - "IndexOutOfBoundsException", "ArrayIndexOutOfBoundsException", - "ClassCastException", "NumberFormatException", "ArithmeticException", - "UnsupportedOperationException", "InterruptedException", - "CloneNotSupportedException", "SecurityException", "StackOverflowError", - "OutOfMemoryError", "AssertionError", - # java.util — collections & core - "Collection", "List", "ArrayList", "LinkedList", "Vector", "Stack", - "Set", "HashSet", "LinkedHashSet", "TreeSet", "SortedSet", "NavigableSet", - "EnumSet", "Map", "HashMap", "LinkedHashMap", "TreeMap", "SortedMap", - "NavigableMap", "Hashtable", "EnumMap", "Properties", "Queue", "Deque", - "ArrayDeque", "PriorityQueue", "Iterator", "ListIterator", "Comparator", - "Optional", "OptionalInt", "OptionalLong", "OptionalDouble", "Collections", - "Arrays", "Objects", "Date", "Calendar", "Random", "UUID", "Scanner", - "StringJoiner", "StringTokenizer", "BitSet", "Spliterator", "Locale", - "NoSuchElementException", "ConcurrentModificationException", - # java.util.stream - "Stream", "IntStream", "LongStream", "DoubleStream", "Collector", - "Collectors", - # java.util.function - "Function", "BiFunction", "Consumer", "BiConsumer", "Supplier", - "Predicate", "BiPredicate", "UnaryOperator", "BinaryOperator", - "IntFunction", "ToIntFunction", "ToLongFunction", "ToDoubleFunction", - # java.util.concurrent - "Callable", "Future", "CompletableFuture", "CompletionStage", "Executor", - "ExecutorService", "Executors", "ScheduledExecutorService", "TimeUnit", - "ConcurrentHashMap", "ConcurrentMap", "CopyOnWriteArrayList", - "BlockingQueue", "CountDownLatch", "Semaphore", "CyclicBarrier", - "AtomicInteger", "AtomicLong", "AtomicBoolean", "AtomicReference", - # java.time - "Instant", "Duration", "Period", "LocalDate", "LocalTime", "LocalDateTime", - "ZonedDateTime", "OffsetDateTime", "ZoneId", "ZoneOffset", "DayOfWeek", - "Month", "Year", "Clock", "DateTimeFormatter", - # java.io / java.nio.file - "IOException", "UncheckedIOException", "FileNotFoundException", "File", - "InputStream", "OutputStream", "Reader", "Writer", "BufferedReader", - "BufferedWriter", "InputStreamReader", "OutputStreamWriter", "FileReader", - "FileWriter", "PrintStream", "PrintWriter", "ByteArrayInputStream", - "ByteArrayOutputStream", "Serializable", "Closeable", "Path", "Paths", - "Files", - # java.math - "BigDecimal", "BigInteger", -}) - - -def _java_collect_type_refs( - node, - source: bytes, - generic: bool, - out: list[tuple[str, str]], - skip: frozenset[str] | None = None, -) -> None: - """Walk a Java type expression; append (name, role) tuples.""" - if node is None: - return - if skip is None: - skip = _java_type_parameters_in_scope(node, source) - t = node.type - if t in ("integral_type", "floating_point_type", "boolean_type", "void_type"): - return - if t == "type_identifier": - name = _read_text(node, source) - if name and name not in skip and name not in _JAVA_BUILTIN_TYPES: - out.append((name, "generic_arg" if generic else "type")) - return - if t == "scoped_type_identifier": - text = _read_text(node, source).rsplit(".", 1)[-1] - if text and text not in _JAVA_BUILTIN_TYPES: - out.append((text, "generic_arg" if generic else "type")) - return - if t == "generic_type": - for c in node.children: - if c.type in ("type_identifier", "scoped_type_identifier"): - text = _read_text(c, source).rsplit(".", 1)[-1] - if ( - text - and text not in _JAVA_BUILTIN_TYPES - and (c.type == "scoped_type_identifier" or text not in skip) - ): - out.append((text, "generic_arg" if generic else "type")) - break - for c in node.children: - if c.type == "type_arguments": - for arg in c.children: - if arg.is_named: - _java_collect_type_refs(arg, source, True, out, skip) - return - if t == "array_type": - for c in node.children: - if c.is_named: - _java_collect_type_refs(c, source, generic, out, skip) - return - if node.is_named: - for c in node.children: - if c.is_named: - _java_collect_type_refs(c, source, generic, out, skip) - - -def _java_annotation_names(declaration_node, source: bytes) -> list[str]: - """Collect annotation names from a Java declaration's `modifiers` child.""" - names: list[str] = [] - modifiers = None - for child in declaration_node.children: - if child.type == "modifiers": - modifiers = child - break - if modifiers is None: - return names - for anno in modifiers.children: - if anno.type not in ("marker_annotation", "annotation"): - continue - name_node = anno.child_by_field_name("name") - if name_node is None: - for sub in anno.children: - if sub.type in ("identifier", "scoped_identifier", "type_identifier"): - name_node = sub - break - if name_node is not None: - text = _read_text(name_node, source).rsplit(".", 1)[-1] - if text: - names.append(text) - return names - - -_GO_PREDECLARED_TYPES = frozenset({ - "bool", "byte", "complex64", "complex128", "error", "float32", "float64", - "int", "int8", "int16", "int32", "int64", "rune", "string", - "uint", "uint8", "uint16", "uint32", "uint64", "uintptr", "any", "comparable", -}) - - -def _go_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None: - """Walk a Go type expression; append (name, role) tuples.""" - if node is None: - return - t = node.type - if t == "type_identifier": - text = _read_text(node, source) - if text and text not in _GO_PREDECLARED_TYPES: - out.append((text, "generic_arg" if generic else "type")) - return - if t == "qualified_type": - text = _read_text(node, source).rsplit(".", 1)[-1] - if text and text not in _GO_PREDECLARED_TYPES: - out.append((text, "generic_arg" if generic else "type")) - return - if t == "generic_type": - type_field = node.child_by_field_name("type") - if type_field is not None: - sub: list[tuple[str, str]] = [] - _go_collect_type_refs(type_field, source, generic, sub) - out.extend(sub) - for c in node.children: - if c.type == "type_arguments": - for arg in c.children: - if arg.is_named: - _go_collect_type_refs(arg, source, True, out) - return - if t in ("pointer_type", "slice_type", "array_type", "map_type", - "channel_type", "parenthesized_type"): - for c in node.children: - if c.is_named: - _go_collect_type_refs(c, source, generic, out) - return - if node.is_named: - for c in node.children: - if c.is_named: - _go_collect_type_refs(c, source, generic, out) -def _rust_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None: - """Walk a Rust type expression; append (name, role) tuples.""" - if node is None: - return - t = node.type - if t == "primitive_type": - return - if t == "type_identifier": - text = _read_text(node, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - return - if t == "scoped_type_identifier": - text = _read_text(node, source).rsplit("::", 1)[-1] - if text: - out.append((text, "generic_arg" if generic else "type")) - return - if t == "generic_type": - name_node = node.child_by_field_name("type") - if name_node is None: - for c in node.children: - if c.type in ("type_identifier", "scoped_type_identifier"): - name_node = c - break - if name_node is not None: - text = _read_text(name_node, source).rsplit("::", 1)[-1] - if text: - out.append((text, "generic_arg" if generic else "type")) - for c in node.children: - if c.type == "type_arguments": - for arg in c.children: - if arg.is_named: - _rust_collect_type_refs(arg, source, True, out) - return - if t in ("reference_type", "pointer_type", "array_type", "tuple_type", "slice_type"): - for c in node.children: - if c.is_named: - _rust_collect_type_refs(c, source, generic, out) - return - if node.is_named: - for c in node.children: - if c.is_named: - _rust_collect_type_refs(c, source, generic, out) +# ── C / C++ type-ref helpers ───────────────────────────────────────────────── -def _php_name_text(node, source: bytes) -> str | None: - """Return the unqualified name text from a PHP `name`/`qualified_name` node.""" - if node is None: - return None - return _read_text(node, source).rsplit("\\", 1)[-1] or None +# ── Scala type-ref helpers ─────────────────────────────────────────────────── -def _php_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None: - """Walk a PHP type expression; append (name, role) tuples.""" - if node is None: - return - t = node.type - if t == "primitive_type": - return - if t == "named_type": - for c in node.children: - if c.type in ("name", "qualified_name"): - text = _php_name_text(c, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - return - return - if t in ("name", "qualified_name"): - text = _php_name_text(node, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - return - if t in ("nullable_type", "union_type", "intersection_type", "optional_type"): - for c in node.children: - if c.is_named: - _php_collect_type_refs(c, source, generic, out) - return - if node.is_named: - for c in node.children: - if c.is_named: - _php_collect_type_refs(c, source, generic, out) - - -def _php_method_return_type_node(method_node): - """Return the named_type/primitive_type node sitting after formal_parameters.""" - saw_params = False - for c in method_node.children: - if c.type == "formal_parameters": - saw_params = True - continue - if saw_params and c.is_named and c.type not in ("compound_statement",): - if c.type in ("named_type", "primitive_type", "nullable_type", - "union_type", "intersection_type", "optional_type"): - return c +def _resolve_name(node, source: bytes, config: LanguageConfig) -> str | None: + """Get the name from a node using config.name_field, falling back to child types.""" + if config.resolve_function_name_fn is not None: + # For C/C++ where the name is inside a declarator + return None # caller handles this separately + n = node.child_by_field_name(config.name_field) + if n: + return _read_text(n, source) + for child in node.children: + if child.type in config.name_fallback_child_types: + return _read_text(child, source) return None -def _kotlin_user_type_name(user_type_node, source: bytes) -> str | None: - """Return the head identifier text from a Kotlin user_type node (without generics).""" - if user_type_node is None: - return None - for c in user_type_node.children: - if c.type == "type_identifier": - text = _read_text(c, source) - return text or None - if c.type == "identifier": - text = _read_text(c, source) - return text or None - if c.type == "simple_user_type": - for sub in c.children: - if sub.type in ("identifier", "type_identifier"): - text = _read_text(sub, source) - return text or None - return None - +# ── Import handlers ─────────────────────────────────────────────────────────── -def _kotlin_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None: - """Walk a Kotlin type expression; append (name, role) tuples.""" - if node is None: - return - t = node.type - if t in ("integral_literal", "boolean_literal"): - return - if t == "user_type": - for c in node.children: - if c.type in ("identifier", "type_identifier"): - text = _read_text(c, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - break - if c.type == "simple_user_type": - for sub in c.children: - if sub.type in ("identifier", "type_identifier"): - text = _read_text(sub, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - break - break - for c in node.children: - if c.type == "type_arguments": - for arg in c.children: - if arg.type == "type_projection": - for sub in arg.children: - if sub.is_named: - _kotlin_collect_type_refs(sub, source, True, out) - elif arg.is_named: - _kotlin_collect_type_refs(arg, source, True, out) - return - if t in ("identifier", "type_identifier"): - text = _read_text(node, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - return - if t in ("nullable_type", "parenthesized_type", "type_reference"): - for c in node.children: - if c.is_named: - _kotlin_collect_type_refs(c, source, generic, out) - return - if node.is_named: - for c in node.children: - if c.is_named: - _kotlin_collect_type_refs(c, source, generic, out) - - -def _kotlin_property_type_node(property_node): - """Find the user_type node within a Kotlin property_declaration.""" - for c in property_node.children: - if c.type == "variable_declaration": - for sub in c.children: - if sub.type in ("user_type", "nullable_type", "type_reference"): - return sub - if c.type in ("user_type", "nullable_type", "type_reference"): - return c - return None - - -def _kotlin_function_return_type_node(func_node): - """Find the return-type node of a Kotlin function_declaration (the type after `: ` post-params).""" - saw_params = False - saw_colon = False - for c in func_node.children: - if c.type == "function_value_parameters": - saw_params = True - continue - if saw_params and c.type == ":": - saw_colon = True - continue - if saw_colon: - if c.is_named: - return c - return None - - -def _swift_declaration_keyword(node) -> str | None: - """Return the leading kind token for a Swift class_declaration: class/struct/enum/extension/actor.""" - for c in node.children: - if not c.is_named and c.type in ("class", "struct", "enum", "extension", "actor"): - return c.type - return None - - -def _swift_pre_scan(root_node, source: bytes) -> tuple[set[str], set[str]]: - """Pre-scan a Swift compilation unit and return (protocol_names, class_like_names).""" - protocols: set[str] = set() - classes: set[str] = set() - stack = [root_node] - while stack: - n = stack.pop() - if n.type == "protocol_declaration": - name_node = n.child_by_field_name("name") - if name_node is None: - for c in n.children: - if c.type == "type_identifier": - name_node = c - break - if name_node is not None: - text = _read_text(name_node, source) - if text: - protocols.add(text) - elif n.type == "class_declaration": - kw = _swift_declaration_keyword(n) - if kw in ("class", "struct", "enum", "actor"): - name_node = n.child_by_field_name("name") - if name_node is not None: - text = _read_text(name_node, source) - if text: - classes.add(text) - stack.extend(n.children) - return protocols, classes - - -def _swift_classify_base(name: str, kind: str | None, is_first: bool, - protocols: set[str], classes: set[str]) -> str: - """Classify a Swift inheritance_specifier entry as `inherits` or `implements`.""" - if name in protocols: - return "implements" - if name in classes: - return "inherits" - # struct/enum/extension/actor cannot inherit a class — all conformances are protocols. - if kind in ("struct", "enum", "extension", "actor"): - return "implements" - # `class`: first entry is conventionally the base class; subsequent are protocols. - return "inherits" if is_first else "implements" - - -def _swift_user_type_name(user_type_node, source: bytes) -> str | None: - """Return the head type_identifier text from a Swift user_type node (without generics).""" - if user_type_node is None: - return None - for c in user_type_node.children: - if c.type == "type_identifier": - text = _read_text(c, source) - return text or None - return None - - -def _swift_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None: - """Walk a Swift type expression; append (name, role) tuples (role 'type' or 'generic_arg').""" - if node is None: - return - t = node.type - if t == "type_annotation": - for c in node.children: - if c.is_named: - _swift_collect_type_refs(c, source, generic, out) - return - if t == "user_type": - for c in node.children: - if c.type == "type_identifier": - text = _read_text(c, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - break - for c in node.children: - if c.type == "type_arguments": - for arg in c.children: - if arg.is_named: - _swift_collect_type_refs(arg, source, True, out) - return - if t == "type_identifier": - text = _read_text(node, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - return - if t in ("optional_type", "implicitly_unwrapped_optional_type", "array_type", - "dictionary_type", "tuple_type"): - for c in node.children: - if c.is_named: - _swift_collect_type_refs(c, source, generic, out) - return - if node.is_named: - for c in node.children: - if c.is_named: - _swift_collect_type_refs(c, source, generic, out) - - -def _swift_property_type_node(property_node): - """Return the type_annotation child of a Swift property_declaration, if any.""" - for c in property_node.children: - if c.type == "type_annotation": - return c - return None - - -def _swift_property_name(property_node, source: bytes) -> str | None: - """Return the bound name of a Swift property (``let x``/``var x = ...``).""" - for c in property_node.children: - if c.type == "pattern": - for sc in c.children: - if sc.type == "simple_identifier": - return _read_text(sc, source) - if c.type == "simple_identifier": - return _read_text(c, source) - return None - - -def _swift_constructor_type(call_node, source: bytes) -> str | None: - """If a Swift call expression is a constructor (``Foo()``), return the type name. - - Only upper-cased callees are treated as types so a free-function call like - ``configure()`` in an initializer is not mistaken for a constructor. - """ - first = call_node.children[0] if call_node.children else None - if first is not None and first.type == "simple_identifier": - text = _read_text(first, source) - if text and text[:1].isupper(): - return text - return None - - -def _swift_receiver_name(recv_node, source: bytes) -> str | None: - """Return the depth-1 receiver name of a Swift member call (``recv.method()``). - - ``vm.update()`` -> ``vm``; ``Type.staticMethod()`` -> ``Type``; - ``Singleton.shared.method()`` -> ``Singleton`` (head of the chain); - ``self.svc.fetch()`` -> ``svc`` (the property the call is reached through). - Returns None for anything deeper, so resolution stays depth-1. - """ - if recv_node is None: - return None - if recv_node.type == "simple_identifier": - return _read_text(recv_node, source) - if recv_node.type == "navigation_expression": - head = recv_node.children[0] if recv_node.children else None - if head is not None and head.type == "simple_identifier": - return _read_text(head, source) - if head is not None and head.type == "self_expression": - for child in recv_node.children: - if child.type == "navigation_suffix": - for sc in child.children: - if sc.type == "simple_identifier": - return _read_text(sc, source) - return None - - -# ── C / C++ type-ref helpers ───────────────────────────────────────────────── - -_C_PRIMITIVE_TYPE_NODES = frozenset({ - "primitive_type", "sized_type_specifier", "auto", "placeholder_type_specifier", -}) - - -def _c_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None: - """Walk a C type expression; append (name, role) tuples for user-defined types. - Skips primitive types and qualifiers; recognises type_identifier.""" - if node is None or node.type in _C_PRIMITIVE_TYPE_NODES: - return - t = node.type - if t == "type_identifier": - text = _read_text(node, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - return - if t in ("pointer_declarator", "reference_declarator", "array_declarator", - "type_qualifier", "type_descriptor", "abstract_pointer_declarator", - "abstract_reference_declarator", "abstract_array_declarator"): - for c in node.children: - if c.is_named: - _c_collect_type_refs(c, source, generic, out) - - -def _cpp_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None: - """Walk a C++ type expression; append (name, role) tuples. - Resolves qualified_identifier tails (std::string → string) and template_type - base + arguments (std::vector → vector + HttpClient as generic_arg).""" - if node is None or node.type in _C_PRIMITIVE_TYPE_NODES: - return - t = node.type - if t == "type_identifier": - text = _read_text(node, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - return - if t == "qualified_identifier": - name_node = node.child_by_field_name("name") - if name_node is not None: - _cpp_collect_type_refs(name_node, source, generic, out) - return - if t == "template_type": - name_node = node.child_by_field_name("name") - if name_node is not None: - text = _read_text(name_node, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - args_node = node.child_by_field_name("arguments") - if args_node is not None: - for c in args_node.children: - if c.is_named: - _cpp_collect_type_refs(c, source, True, out) - return - if t in ("type_descriptor", "pointer_declarator", "reference_declarator", - "array_declarator", "type_qualifier", "abstract_pointer_declarator", - "abstract_reference_declarator", "abstract_array_declarator"): - for c in node.children: - if c.is_named: - _cpp_collect_type_refs(c, source, generic, out) - - -# ── Scala type-ref helpers ─────────────────────────────────────────────────── - -def _scala_collect_type_refs(node, source: bytes, generic: bool, out: list[tuple[str, str]]) -> None: - """Walk a Scala type expression; append (name, role) tuples. - Handles type_identifier, generic_type (List[T]), and common type wrappers.""" - if node is None: - return - t = node.type - if t == "type_identifier": - text = _read_text(node, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - return - if t == "generic_type": - base = node.child_by_field_name("type") - if base is None: - for c in node.children: - if c.type == "type_identifier": - base = c - break - if base is not None and base.type == "type_identifier": - text = _read_text(base, source) - if text: - out.append((text, "generic_arg" if generic else "type")) - for c in node.children: - if c.type == "type_arguments": - for arg in c.children: - if arg.is_named: - _scala_collect_type_refs(arg, source, True, out) - return - if t in ("compound_type", "infix_type", "function_type", "tuple_type", - "annotated_type", "projected_type"): - for c in node.children: - if c.is_named: - _scala_collect_type_refs(c, source, generic, out) - - -def _python_collect_param_refs(params_node, source: bytes) -> list[tuple[str, str]]: - """Collect type refs from each typed parameter under a `parameters` node.""" - out: list[tuple[str, str]] = [] - if params_node is None: - return out - for child in params_node.children: - if child.type in ("typed_parameter", "typed_default_parameter"): - type_node = child.child_by_field_name("type") - _python_collect_type_refs(type_node, source, False, out) - return out - - -def _python_param_names(params_node, source: bytes) -> set[str]: - """Plain parameter identifiers declared on a Python `parameters` node. - - Covers positional/keyword params plus `*args` / `**kwargs` and typed or - default forms — anything that binds a local name the function body can shadow - a module-level definition with. - """ - out: set[str] = set() - if params_node is None: - return out - for child in params_node.children: - if child.type == "identifier": - out.add(_read_text(child, source)) - elif child.type in ( - "typed_parameter", - "default_parameter", - "typed_default_parameter", - "list_splat_pattern", - "dictionary_splat_pattern", - ): - # The bound name is the first identifier child (the rest is type/default). - name_n = child.child_by_field_name("name") - if name_n is None: - name_n = next( - (c for c in child.children if c.type == "identifier"), None - ) - if name_n is not None: - out.add(_read_text(name_n, source)) - return out - - -def _python_collect_assignment_targets(node, source: bytes, out: set[str]) -> None: - """Identifiers bound as `pattern` targets under a Python AST subtree. - - Recurses through `pattern_list` / `tuple_pattern` / `list_pattern` so tuple - unpacking (`a, b = ...`, `for a, b in ...`) contributes every bound name. - """ - if node is None: - return - if node.type == "identifier": - out.add(_read_text(node, source)) - return - if node.type in ("pattern_list", "tuple_pattern", "list_pattern"): - for c in node.children: - _python_collect_assignment_targets(c, source, out) - - -def _python_local_bound_names(func_def_node, source: bytes) -> set[str]: - """Names bound LOCALLY inside a Python function: parameters plus assignment, - `for`, `with ... as`, and comprehension targets. - - Used by the indirect-dispatch guard to reject a call-argument identifier that - is a parameter or a local binding — it names a local value, not the module- - level function/class that happens to share the name. Nested `function_definition` - and `class_definition` subtrees are NOT descended into: their bindings belong - to a different scope. - """ - bound: set[str] = set() - bound |= _python_param_names(func_def_node.child_by_field_name("parameters"), source) - - def walk(n) -> None: - for child in n.children: - t = child.type - if t in ("function_definition", "class_definition", "lambda"): - continue # inner scope — its bindings are not this function's locals - if t == "assignment": - _python_collect_assignment_targets( - child.child_by_field_name("left"), source, bound - ) - elif t in ("for_statement", "for_in_clause"): - _python_collect_assignment_targets( - child.child_by_field_name("left"), source, bound - ) - elif t == "with_statement": - for item in child.children: - if item.type == "with_clause": - for wi in item.children: - if wi.type == "with_item": - alias = wi.child_by_field_name("alias") - _python_collect_assignment_targets(alias, source, bound) - elif t == "named_expression": # walrus := - _python_collect_assignment_targets( - child.child_by_field_name("name"), source, bound - ) - walk(child) - - body = func_def_node.child_by_field_name("body") - if body is not None: - walk(body) - return bound - - -def _python_module_bound_names(root, source: bytes) -> set[str]: - """Names rebound by assignment at MODULE scope (top-level `x = ...`, `for`, walrus). - - The module-scope analogue of the per-function shadow set: a dispatch-table value - whose name is reassigned to data at module level (`handler = build()`) names that - value, not a same-named function, so it must not manufacture an indirect edge. - Function and class bodies are not descended into — their bindings are local. - """ - bound: set[str] = set() - - def walk(n) -> None: - for child in n.children: - t = child.type - if t in ("function_definition", "class_definition", "lambda"): - continue # inner scope — not a module-level binding - if t == "assignment": - _python_collect_assignment_targets( - child.child_by_field_name("left"), source, bound - ) - elif t in ("for_statement", "for_in_clause"): - _python_collect_assignment_targets( - child.child_by_field_name("left"), source, bound - ) - elif t == "named_expression": # walrus := - _python_collect_assignment_targets( - child.child_by_field_name("name"), source, bound - ) - walk(child) - - walk(root) - return bound - - -_JS_SCOPE_BOUNDARY = frozenset({ - "function_declaration", "function_expression", "function", "arrow_function", - "method_definition", "class_declaration", "class", "generator_function", - "generator_function_declaration", -}) - - -def _js_collect_pattern_idents(node, source: bytes, bound: set) -> None: - """Collect binding identifier names from a JS/TS pattern (a parameter, or a - declarator LHS). Recurses through destructuring (object/array patterns, rest) - but never into the default-value side of `x = default` or a type annotation, - so only names actually bound by the pattern are collected.""" - t = node.type - if t in ("identifier", "shorthand_property_identifier_pattern"): - bound.add(_read_text(node, source)) - return - if t == "type_annotation": - return # `(h: Handler)` — Handler is a type, not a bound name - if t == "assignment_pattern": # `x = default` — only x is bound - left = node.child_by_field_name("left") - if left is not None: - _js_collect_pattern_idents(left, source, bound) - return - if t == "pair_pattern": # `{ a: localName }` — localName is bound - val = node.child_by_field_name("value") - if val is not None: - _js_collect_pattern_idents(val, source, bound) - return - for c in node.children: - if c.is_named: - _js_collect_pattern_idents(c, source, bound) - - -def _js_local_bound_names(func_node, source: bytes) -> set[str]: - """Names bound locally inside a JS/TS function: parameters plus `const`/`let`/ - `var` declarator targets. Mirrors `_python_local_bound_names`: an argument that - is a parameter or local binding names a local value, not a same-named module - function, so it must not manufacture an indirect_call edge. Nested function and - class scopes are not descended into.""" - bound: set[str] = set() - params = func_node.child_by_field_name("parameters") - if params is not None: - _js_collect_pattern_idents(params, source, bound) - - def walk(n) -> None: - for c in n.children: - if c.type in _JS_SCOPE_BOUNDARY: - continue # inner scope — its bindings are not this function's locals - if c.type == "variable_declarator": - name = c.child_by_field_name("name") - if name is not None: - _js_collect_pattern_idents(name, source, bound) - walk(c) - - body = func_node.child_by_field_name("body") - if body is not None: - walk(body) - return bound - - -def _js_module_bound_names(root, source: bytes) -> set[str]: - """Module-scope names rebound to NON-function data (`const X = {...}`, `let y = 5`). - - The JS/TS module-scope shadow set. Unlike the per-function set, a declarator - whose value is itself a function (`const cb = () => {}`) is EXCLUDED: that name - IS a callable we want dispatch tables to resolve to, not a data shadow. - """ - bound: set[str] = set() - - def walk(n) -> None: - for c in n.children: - if c.type in _JS_SCOPE_BOUNDARY: - continue - if c.type == "variable_declarator": - value = c.child_by_field_name("value") - if value is None or value.type not in _JS_FUNCTION_VALUE_TYPES: - name = c.child_by_field_name("name") - if name is not None: - _js_collect_pattern_idents(name, source, bound) - walk(c) - - walk(root) - return bound - - -def _js_dispatch_value_idents(coll_node): - """Yield identifier value-nodes of a JS/TS object/array literal that are - function-reference candidates: object property VALUES and shorthand properties - (`{ handler }`), and array elements. Keys and inline methods are not references.""" - if coll_node.type == "object": - for c in coll_node.children: - if c.type == "pair": - val = c.child_by_field_name("value") - if val is not None and val.type == "identifier": - yield val - elif c.type == "shorthand_property_identifier": - yield c - else: # array - for el in coll_node.children: - if el.type == "identifier": - yield el - - -def _resolve_name(node, source: bytes, config: LanguageConfig) -> str | None: - """Get the name from a node using config.name_field, falling back to child types.""" - if config.resolve_function_name_fn is not None: - # For C/C++ where the name is inside a declarator - return None # caller handles this separately - n = node.child_by_field_name(config.name_field) - if n: - return _read_text(n, source) - for child in node.children: - if child.type in config.name_fallback_child_types: - return _read_text(child, source) - return None - - -def _find_body(node, config: LanguageConfig): - """Find the body node using config.body_field, falling back to child types.""" - b = node.child_by_field_name(config.body_field) - if b: - return b - for child in node.children: - if child.type in config.body_fallback_child_types: - return child - return None - - -# ── Import handlers ─────────────────────────────────────────────────────────── - -def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: +def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: t = node.type if t == "import_statement": for child in node.children: if child.type in ("dotted_name", "aliased_import"): raw = _read_text(child, source) - module_name = raw.split(" as ")[0].strip().lstrip(".") + raw_module, _, raw_alias = raw.partition(" as ") + module_name = raw_module.strip().lstrip(".") tgt_nid = _make_id(module_name) - edges.append({ + edge = { "source": file_nid, "target": tgt_nid, "relation": "imports", @@ -1795,11 +327,19 @@ def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, s "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, - }) + } + if raw_alias: + # `import pkg.mod as alias` binds the local name `alias`, not + # `mod`'s own stem, to the module -- stash it so the cross-file + # member-call resolver can match `alias.func()` against this + # edge instead of dropping it (#2082). + edge["local_alias"] = raw_alias.strip() + edges.append(edge) elif t == "import_from_statement": module_node = node.child_by_field_name("module_name") if module_node: raw = _read_text(module_node, source) + target_path: "Path | None" = None if raw.startswith("."): # Relative import - resolve to full path so IDs match file node IDs dots = len(raw) - len(raw.lstrip(".")) @@ -1808,10 +348,11 @@ def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, s for _ in range(dots - 1): base = base.parent rel = (module_name.replace(".", "/") + ".py") if module_name else "__init__.py" - tgt_nid = _make_id(str(base / rel)) + target_path = base / rel + tgt_nid = _make_id(str(target_path)) else: tgt_nid = _make_id(raw) - edges.append({ + edge = { "source": file_nid, "target": tgt_nid, "relation": "imports_from", @@ -1820,35 +361,22 @@ def _import_python(node, source: bytes, file_nid: str, stem: str, edges: list, s "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, - }) - - -def _resolve_js_import_target(raw: str, str_path: str) -> "tuple[str, Path | None] | None": - """Resolve a JS/TS import path string to (target_nid, resolved_path). - - Handles relative paths, tsconfig path aliases, workspace packages, and - bare/scoped imports. - Returns None if `raw` is empty. - """ - if not raw: - return None - resolved_path = _resolve_js_module_path(raw, Path(str_path).parent) - if resolved_path is not None: - return _make_id(str(resolved_path)), resolved_path - module_name = raw.split("/")[-1] - if not module_name: - return None - # Unresolved: relative/absolute, tsconfig-alias and workspace resolution have - # all run and failed, so this is an external package (or a dangling local - # path). Namespace the id with the "ref" prefix — the J-4 convention already - # used for tsconfig `extends`/`$ref` externals — so it can NEVER collapse to - # the same _make_id as a local file/symbol node. Without it, the bare - # last-segment id (e.g. "tailwindcss/colors" -> "colors") collides with any - # unrelated local file of that stem via build.py's pre-migration alias index, - # producing a confident (EXTRACTED) cross-language phantom imports_from edge - # (#1638). The ref-namespaced target has no node, so build drops it as an - # external reference — the correct outcome for a third-party import. - return _make_id("ref", raw), None + } + # Stamp the resolved target file (mirroring _import_js, #1814) so + # the #2169 remap pass can canonicalize this edge's target on an + # incremental run where the target file itself is not in the + # batch — without it the target keeps an absolute-path-derived id + # that matches no node in the merged graph and dangles (#2213). + # Existence-gated: a speculative import of a nonexistent sibling + # must stay dangling, exactly as before. The stamp is transient + # and popped before graph.json ships. + if target_path is not None: + try: + if target_path.is_file(): + edge["target_file"] = str(target_path) + except OSError: + pass + edges.append(edge) def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: @@ -1882,7 +410,7 @@ def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_p resolved = _resolve_js_import_target(raw, str_path) if resolved is not None: tgt_nid, resolved_path = resolved - edges.append({ + edge = { "source": file_nid, "target": tgt_nid, "relation": "imports_from", @@ -1891,7 +419,15 @@ def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_p "source_file": str_path, "source_location": f"L{node.start_point[0] + 1}", "weight": 1.0, - }) + } + # Stamp the resolved target file so a same-basename cross-extension + # sibling (foo.ts importing/re-exporting ./foo.mjs) keys its target salt + # by the TARGET's file rather than the importer's. Both files collapse to + # the base id `foo`; without this the salted lookup mis-points the target + # back onto the importer's own variant, a phantom self-loop (#1814). + if resolved_path is not None: + edge["target_file"] = str(resolved_path) + edges.append(edge) # Emit symbol-level edges for named imports/re-exports from local/aliased files. # e.g. `import { Foo, type Bar } from './bar'` → file → Foo, file → Bar (EXTRACTED) @@ -1924,6 +460,12 @@ def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_p "source_file": str_path, "source_location": f"L{line}", "weight": 1.0, + # Which file this symbol target was synthesized + # from, so the id-remap post-pass can repoint a + # target the candidates rewrite never learns — + # a barrel defines no symbols (#1983). Transient, + # stripped at build like the #1814 stamp. + "target_file": str(resolved_path), }) else: # Handle: import { Foo, type Bar } from './bar' @@ -1945,76 +487,11 @@ def _import_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_p "source_file": str_path, "source_location": f"L{line}", "weight": 1.0, + # See the re_exports stamp above (#1983). + "target_file": str(resolved_path), }) -def _dynamic_import_js(node, source: bytes, caller_nid: str, str_path: str, edges: list, - seen_dyn_pairs: set) -> bool: - """Detect dynamic import() calls in JS/TS and emit imports_from edges. - - Handles patterns like: - await import('./foo.js') - import('./foo.js').then(...) - const m = await import(`./foo`) - - Returns True if the node was a dynamic import (caller should skip normal call handling). - """ - # Dynamic import is a call_expression whose function child is the keyword "import". - # tree-sitter-typescript parses `import('...')` as call_expression with first child - # being an "import" token (type="import"). - func_node = node.child_by_field_name("function") - if func_node is None: - # Fallback: check first child directly (some TS versions) - if node.children and _read_text(node.children[0], source) == "import": - func_node = node.children[0] - else: - return False - if _read_text(func_node, source) != "import": - return False - - # Extract the module path from the arguments - args = node.child_by_field_name("arguments") - if args is None: - return True # It's an import() but no args — skip - for arg in args.children: - if arg.type == "template_string": - # Skip dynamic template literals — path can't be statically resolved - if any(c.type == "template_substitution" for c in arg.children): - break - raw = _read_text(arg, source).strip("`") - elif arg.type == "string": - raw = _read_text(arg, source).strip("'\" ") - else: - continue - if not raw: - break - # Resolve path using the same logic as static imports. - resolved = _resolve_js_import_target(raw, str_path) - if resolved is None: - break - tgt_nid, _ = resolved - pair = (caller_nid, tgt_nid) - if pair not in seen_dyn_pairs: - seen_dyn_pairs.add(pair) - edges.append({ - "source": caller_nid, - "target": tgt_nid, - # A deferred `import(...)` is a real dependency, so keep it as an - # `imports_from` edge (visible in the graph) but mark it `deferred` - # so find_import_cycles does not treat it as a static import and - # report a phantom file cycle (#1241). - "relation": "imports_from", - "context": "import", - "deferred": True, - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - "weight": 1.0, - }) - break - return True - - def _import_java(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: def _walk_scoped(n) -> str: parts: list[str] = [] @@ -2054,20 +531,6 @@ def _walk_scoped(n) -> str: break -def _resolve_c_include_path(raw: str, str_path: str) -> "Path | None": - """Resolve a quoted #include path to a real file on disk. - - Searches relative to the including file's directory. Returns None for - system headers (<...>) or paths that don't exist on disk. - """ - if not raw: - return None - candidate = (Path(str_path).parent / raw).resolve() - if candidate.is_file(): - return candidate - return None - - def _import_c(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: for child in node.children: if child.type in ("string_literal", "system_lib_string", "string"): @@ -2227,713 +690,27 @@ def _get_c_func_name(node, source: bytes) -> str | None: return None -def _get_cpp_func_name(node, source: bytes) -> str | None: - """Recursively unwrap declarator to find the innermost identifier (C++).""" - if node.type == "identifier": - return _read_text(node, source) - if node.type in ("field_identifier", "destructor_name", "operator_name"): - return _read_text(node, source) - if node.type == "qualified_identifier": - # An out-of-class DEFINITION (`void Foo::bar() {}`) carries a - # qualified_identifier declarator. Retaining the `Foo::` qualifier makes - # _make_id(stem, "Foo::bar") normalize to the same id as the in-class - # member _make_id(class_nid, "bar"), so the decl in Foo.h and the def in - # Foo.cpp resolve to ONE method node instead of two (#1547). The full - # qualified text also handles nested scopes (`A::B::bar`). Free functions - # never have a qualified_identifier here, so their bare-name ids are - # unchanged; only qualified definitions shift onto their owning class. - return _read_text(node, source) - decl = node.child_by_field_name("declarator") - if decl: - return _get_cpp_func_name(decl, source) - for child in node.children: - if child.type == "identifier": - return _read_text(child, source) - return None - - -def _cpp_declarator_name(node, source: bytes) -> str | None: - """Return the bare variable name from a C++ declaration declarator, unwrapping - pointer/reference/init wrappers (``*f``, ``&r``, ``f = Foo()``). Returns None - for anything that isn't a plain named local (arrays, function pointers, - structured bindings) so the type table never records a guessed receiver.""" - t = node.type - if t == "identifier": - return _read_text(node, source) - if t in ("pointer_declarator", "reference_declarator", "init_declarator"): - inner = node.child_by_field_name("declarator") - if inner is None: - for c in node.children: - if c.type in ("identifier", "pointer_declarator", - "reference_declarator"): - inner = c - break - if inner is not None: - return _cpp_declarator_name(inner, source) - return None - - -def _cpp_local_var_types(body_node, source: bytes, table: dict[str, str]) -> None: - """Collect ``var -> ClassName`` from local variable declarations in a C++ - function body, for receiver-type inference in the cross-file member-call pass - (#1547). Handles ``Foo f;``, ``Foo* f;``, ``Foo *f = ...;``, ``Foo f = Foo();``. - - Only a class-like (``type_identifier``/``qualified_identifier``) type with a - single named declarator is recorded — PRECISION over recall: a built-in type - (``int x``), an ambiguous multi-declarator line, or an un-nameable declarator - contributes nothing rather than a guess. A qualified type ``ns::Foo`` records - its simple tail ``Foo`` so it keys to the type's definition node label. - """ - stack = [body_node] - while stack: - n = stack.pop() - if n.type in ("function_definition", "lambda_expression"): - # Don't descend into a nested function/lambda: its locals are scoped - # away and would pollute this body's table. - if n is not body_node: - continue - if n.type == "declaration": - type_node = n.child_by_field_name("type") - if type_node is not None and type_node.type in ( - "type_identifier", "qualified_identifier" - ): - type_name = _read_text(type_node, source).split("::")[-1].strip() - declarators = [ - c for c in n.children - if c.type in ("identifier", "pointer_declarator", - "reference_declarator", "init_declarator") - ] - # A single declarator only: `Foo a, b;` is ambiguous to attribute - # to one receiver name cleanly, so skip multi-declarator lines. - if type_name and type_name[:1].isupper() and len(declarators) == 1: - var = _cpp_declarator_name(declarators[0], source) - if var and var not in table: - table[var] = type_name - for c in n.children: - stack.append(c) - - -def _swift_local_var_types(body_node, source: bytes, table: dict[str, str]) -> None: - """Collect ``var -> Type`` from local ``let``/``var`` bindings in a Swift - function body, so a member call on the local (``x.method()``) resolves to Type - in the cross-file member-call pass (#1604). - - Two initializer shapes are recorded, PRECISION over recall: - - a constructor call ``let x = Type()`` (``_swift_constructor_type``); - - a static-member access ``let x = Type.shared`` (a navigation_expression - with an upper-cased head) — the singleton-cached-into-a-local idiom, one - of the most common Swift call patterns and previously resolved to nothing. - Nested function declarations are not descended into (their locals are scoped - away); the first binding for a name wins, so a class property of the same name - already in the table is not overwritten. - """ - stack = [body_node] - while stack: - n = stack.pop() - if n.type == "function_declaration" and n is not body_node: - continue - if n.type == "property_declaration": - prop_type: str | None = None - for child in n.children: - if child.type == "call_expression": - prop_type = _swift_constructor_type(child, source) - break - if child.type == "navigation_expression": - head = child.children[0] if child.children else None - if head is not None and head.type == "simple_identifier": - htext = _read_text(head, source) - if htext and htext[:1].isupper(): - prop_type = htext - break - name = _swift_property_name(n, source) - if name and prop_type and name not in table: - table[name] = prop_type - for c in n.children: - stack.append(c) - - -def _csharp_member_type_table(root, source: bytes) -> dict[str, str]: - """Collect ``name -> TypeName`` for C# receiver typing (#1609): class fields, - properties, method parameters, and local variable declarations. - - File-scoped, first-binding-wins (like the C++ table): a field declared once at - class scope is visible to every method's `field.Method()`, and a param/local - shadowing the same name is a conservative approximation graphify already accepts - for receiver typing. Only a resolvable, non-`var` type name is recorded; `var` - without a `new T()` initializer, and predefined/lower-cased primitives, are - skipped (precision over recall — an untypable receiver is left for the resolver - to drop rather than guess). `var v = new T()` is typed from the object-creation. - """ - table: dict[str, str] = {} - - def _typed(type_node) -> str | None: - info = _read_csharp_type_name(type_node, source) - if not info: - return None - name = info[0] - # A genuine C# class name is Pascal-cased; skip predefined primitives - # (int/bool/string) which never own a resolvable method definition here. - return name if name and name[:1].isupper() else None - - def _decl_names(var_decl): - for c in var_decl.children: - if c.type == "variable_declarator": - nm = c.child_by_field_name("name") or next( - (g for g in c.children if g.type == "identifier"), None) - if nm is not None: - yield _read_text(nm, source), c - - def _new_type(declarator) -> str | None: - # `var v = new Server()` — recover the type from the object_creation_expression. - for g in declarator.children: - if g.type == "object_creation_expression": - return _typed(g.child_by_field_name("type")) - return None - - stack = [root] - while stack: - n = stack.pop() - t = n.type - if t in ("field_declaration", "local_declaration_statement"): - vd = next((c for c in n.children if c.type == "variable_declaration"), None) - if vd is not None: - type_node = vd.child_by_field_name("type") - declared = _typed(type_node) - for name, decl in _decl_names(vd): - resolved = declared or _new_type(decl) - if name and resolved and name not in table: - table[name] = resolved - elif t == "property_declaration": - nm = n.child_by_field_name("name") - resolved = _typed(n.child_by_field_name("type")) - if nm is not None and resolved: - pname = _read_text(nm, source) - if pname not in table: - table[pname] = resolved - elif t == "parameter": - nm = n.child_by_field_name("name") - resolved = _typed(n.child_by_field_name("type")) - if nm is not None and resolved: - pname = _read_text(nm, source) - if pname not in table: - table[pname] = resolved - for c in n.children: - stack.append(c) - return table - - -def _ts_receiver_type_table(root, source: bytes, table: dict[str, str]) -> None: - """Add TS/JS receiver bindings to ``table`` (name -> TypeName), for member-call - resolution beyond the constructor-injected `this.field` case (#1630): - - * local ``const/let/var x = new Foo()`` -> ``x: Foo`` (Pattern A); - * a type-annotated parameter ``(svc: Svc)`` -> ``svc: Svc`` (Pattern B), so a - call on the param — including inside a returned closure — resolves. - - File-scoped, first-binding-wins (merged into the constructor-injection table, - which is populated first and therefore wins on a name clash). Only a bare - ``type_identifier`` (a single class/interface name) is recorded — an array, - union, generic, qualified, or predefined type is skipped (precision over - recall, matching the receiver-typed resolvers for Swift/C#/C++).""" - def _bare_type_ident(type_annotation): - # type_annotation -> ": T"; accept only a single type_identifier child. - idents = [c for c in type_annotation.children if c.type == "type_identifier"] - others = [c for c in type_annotation.children - if c.is_named and c.type not in ("type_identifier",)] - if len(idents) == 1 and not others: - return _read_text(idents[0], source) - return None - - stack = [root] - while stack: - n = stack.pop() - t = n.type - if t == "variable_declarator": - name_n = n.child_by_field_name("name") - value = n.child_by_field_name("value") - if (name_n is not None and name_n.type == "identifier" - and value is not None and value.type == "new_expression"): - ctor = value.child_by_field_name("constructor") - if ctor is not None and ctor.type in ("identifier", "type_identifier"): - name = _read_text(name_n, source) - tname = _read_text(ctor, source) - if name and tname and name not in table: - table[name] = tname - elif t == "required_parameter" or t == "optional_parameter": - pat = n.child_by_field_name("pattern") - ann = n.child_by_field_name("type") - if pat is not None and pat.type == "identifier" and ann is not None: - tname = _bare_type_ident(ann) - name = _read_text(pat, source) - if name and tname and name not in table: - table[name] = tname - for c in n.children: - stack.append(c) - - -def _objc_local_var_types(body_node, source: bytes, table: dict[str, str]) -> None: - """Collect ``var -> ClassName`` from ObjC local declarations (``Foo *f = ...;``) - in a method body, for receiver typing in the cross-file message-send pass - (#1556). Only a capitalized ``type_identifier`` with a single named declarator - is recorded; a built-in/lower-cased type or an un-nameable declarator is skipped - (precision over recall). Reuses the C++ declarator unwrapper (identical grammar). - """ - stack = [body_node] - while stack: - n = stack.pop() - if n.type == "method_definition" and n is not body_node: - continue - if n.type == "declaration": - type_node = n.child_by_field_name("type") - if type_node is None: - for c in n.children: - if c.type == "type_identifier": - type_node = c - break - if type_node is not None and type_node.type == "type_identifier": - type_name = _read_text(type_node, source).strip() - declarators = [ - c for c in n.children - if c.type in ("identifier", "pointer_declarator", "init_declarator") - ] - if type_name and type_name[:1].isupper() and len(declarators) == 1: - var = _cpp_declarator_name(declarators[0], source) - if var and var not in table: - table[var] = type_name - for c in n.children: - stack.append(c) - - # ── JS/TS extra walk for arrow functions ────────────────────────────────────── -def _find_require_call(value_node): - """Return the call_expression node if `value_node` is a `require(...)` call - or `require(...).x` member access. Otherwise None.""" - if value_node is None: - return None - if value_node.type == "call_expression": - fn = value_node.child_by_field_name("function") - if fn is not None and fn.type == "identifier": - return value_node - if value_node.type == "member_expression": - obj = value_node.child_by_field_name("object") - return _find_require_call(obj) - return None - - -def _require_imports_js(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str) -> bool: - """Detect CommonJS require imports inside lexical_declaration / variable_declaration. - - Handles three patterns: - const { foo, bar } = require('./mod') → file → mod (imports_from), file → foo, file → bar - const mod = require('./mod') → file → mod (imports_from) - const x = require('./mod').y → file → mod (imports_from), file → y - - Returns True if any require import was found. - """ - if node.type not in ("lexical_declaration", "variable_declaration"): - return False - found = False - for child in node.children: - if child.type != "variable_declarator": - continue - value = child.child_by_field_name("value") - call = _find_require_call(value) - if call is None: - continue - fn = call.child_by_field_name("function") - if fn is None or _read_text(fn, source) != "require": - continue - args = call.child_by_field_name("arguments") - if args is None: - continue - raw = None - for arg in args.children: - if arg.type == "string": - raw = _read_text(arg, source).strip("'\"` ") - break - if not raw: - continue - resolved = _resolve_js_import_target(raw, str_path) - if resolved is None: - continue - tgt_nid, resolved_path = resolved - line = node.start_point[0] + 1 - edges.append({ - "source": file_nid, - "target": tgt_nid, - "relation": "imports_from", - "context": "import", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{line}", - "weight": 1.0, - }) - found = True - - # Symbol-level edges for destructured / accessor binders. - target_stem = _file_stem(resolved_path) if resolved_path is not None else None - name_node = child.child_by_field_name("name") - sym_names: list[str] = [] - if name_node is not None and name_node.type == "object_pattern": - # `const { a, b: alias } = require('./m')` — emit edges for each property key - for prop in name_node.children: - if prop.type == "shorthand_property_identifier_pattern": - sym_names.append(_read_text(prop, source)) - elif prop.type == "pair_pattern": - key = prop.child_by_field_name("key") - if key is not None: - sym_names.append(_read_text(key, source)) - elif value is not None and value.type == "member_expression": - # `const x = require('./m').y` — symbol is the property accessed - prop = value.child_by_field_name("property") - if prop is not None: - sym_names.append(_read_text(prop, source)) - if target_stem is not None: - for sym in sym_names: - edges.append({ - "source": file_nid, - "target": _make_id(target_stem, sym), - "relation": "imports", - "context": "import", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{line}", - "weight": 1.0, - }) - return found - # Node types whose value is a callable, for the JS/TS assignment / class-field # / function-expression forms below. Older tree-sitter-javascript grammars # label a function expression `function`; current ones use `function_expression`. -_JS_FUNCTION_VALUE_TYPES = frozenset({"arrow_function", "function_expression", "function", "generator_function"}) -def _js_member_assignment_target(left, source: bytes): - """Classify the symbol an `assignment_expression` LHS defines when its RHS - is a function. Returns (kind, owner_name, member_name) or None. +# ── TS extra walk for namespace / module declarations ───────────────────────── - this.foo = fn → ("this", None, "foo") - exports.foo = fn → ("exports", None, "foo") - module.exports.foo = fn → ("exports", None, "foo") - Foo.prototype.bar = fn → ("prototype", "Foo", "bar") - Any other shape (an arbitrary `obj.x = fn`) returns None and is skipped — - capturing those would reintroduce the bare-named / phantom-god-node class - of bug the module-level scope guard (#1077) exists to prevent. - """ - if left is None or left.type != "member_expression": - return None - prop = left.child_by_field_name("property") - if prop is None: - return None - member_name = _read_text(prop, source) - if not member_name: - return None - obj = left.child_by_field_name("object") - if obj is None: - return None - if obj.type == "this": - return ("this", None, member_name) - if obj.type == "identifier": - if _read_text(obj, source) == "exports": - return ("exports", None, member_name) - return None - if obj.type == "member_expression": - # module.exports.X or Foo.prototype.X - inner_obj = obj.child_by_field_name("object") - inner_prop = obj.child_by_field_name("property") - if inner_obj is None or inner_prop is None: - return None - inner_prop_name = _read_text(inner_prop, source) - if inner_obj.type == "identifier": - inner_obj_name = _read_text(inner_obj, source) - if inner_obj_name == "module" and inner_prop_name == "exports": - return ("exports", None, member_name) - if inner_prop_name == "prototype": - return ("prototype", inner_obj_name, member_name) - return None +# ── C# extra walk for namespace declarations ────────────────────────────────── -def _js_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: str, - nodes: list, edges: list, seen_ids: set, function_bodies: list, - parent_class_nid: str | None, add_node_fn, add_edge_fn, - callable_def_nids: set | None = None, - local_bound_names: dict | None = None) -> bool: - """Handle lexical_declaration (arrow functions, CJS requires, module-level const literals) for JS/TS. Returns True if handled.""" - # CommonJS / prototype member assignments whose value is a function: - # exports.X = () => {} → file-contained function X() - # module.exports.X = fn → file-contained function X() - # Foo.prototype.bar = fn → method bar() owned by Foo - # (`this.X = fn` lives inside a function body, which is not recursed here; - # it is captured at the enclosing function — see the function branch.) - if node.type == "expression_statement": - assign = next((c for c in node.children - if c.type == "assignment_expression"), None) - if assign is not None: - value = assign.child_by_field_name("right") - if value is not None and value.type in _JS_FUNCTION_VALUE_TYPES: - target = _js_member_assignment_target( - assign.child_by_field_name("left"), source) - if target is not None: - kind, owner_name, member_name = target - line = node.start_point[0] + 1 - handled = False - if kind == "exports": - nid = _make_id(stem, member_name) - add_node_fn(nid, f"{member_name}()", line) - add_edge_fn(file_nid, nid, "contains", line) - handled = True - elif kind == "prototype": - owner_nid = _make_id(stem, owner_name) - nid = _make_id(owner_nid, member_name) - add_node_fn(nid, f".{member_name}()", line) - add_edge_fn(owner_nid, nid, "method", line) - handled = True - if handled: - if callable_def_nids is not None: - callable_def_nids.add(nid) # CJS/prototype fn is callable - if local_bound_names is not None: - local_bound_names[nid] = _js_local_bound_names(value, source) - body = value.child_by_field_name("body") - if body: - function_bodies.append((nid, body)) - return True - - # Class fields whose value is a function: - # class C { handler = () => {} } → method handler() owned by C - # Reaches here with parent_class_nid set because class bodies are recursed - # with the class nid as parent. - if parent_class_nid and node.type in ("field_definition", "public_field_definition"): - prop = node.child_by_field_name("property") or node.child_by_field_name("name") - value = node.child_by_field_name("value") - if (prop is not None and value is not None - and value.type in _JS_FUNCTION_VALUE_TYPES): - field_name = _read_text(prop, source) - if field_name: - line = node.start_point[0] + 1 - nid = _make_id(parent_class_nid, field_name) - add_node_fn(nid, f".{field_name}()", line) - add_edge_fn(parent_class_nid, nid, "method", line) - if callable_def_nids is not None: - callable_def_nids.add(nid) # arrow class-field is callable - if local_bound_names is not None: - local_bound_names[nid] = _js_local_bound_names(value, source) - body = value.child_by_field_name("body") - if body: - function_bodies.append((nid, body)) - return True +# ── Swift extra walk for enum cases ────────────────────────────────────────── - if node.type in ("lexical_declaration", "variable_declaration"): - # CJS require imports — emit edges, do not block other lexical_declaration handling - require_found = _require_imports_js(node, source, file_nid, stem, edges, str_path) - - # Scope guard (#1077): only emit nodes for module-level declarations. - # Without this, `const x = ...` inside an arrow callback (e.g. inside - # `describe(() => { const set = new Set(...) })`) emits a bare-named - # node, and the same name collides across unrelated files producing - # phantom god-nodes. Bodies of arrow functions are walked separately - # via function_bodies, so we never need to emit nodes for locals here. - parent = node.parent - is_module_level = parent is not None and ( - parent.type == "program" - or (parent.type == "export_statement" - and parent.parent is not None - and parent.parent.type == "program") - ) - # Arrow function declarations and module-level const literals (lexical_declaration only) - arrow_found = False - const_found = False - if node.type == "lexical_declaration" and is_module_level: - for child in node.children: - if child.type == "variable_declarator": - value = child.child_by_field_name("value") - if value and value.type in _JS_FUNCTION_VALUE_TYPES: - # `const f = () => {}` and `const f = function(){}` - name_node = child.child_by_field_name("name") - if name_node: - func_name = _read_text(name_node, source) - line = child.start_point[0] + 1 - func_nid = _make_id(stem, func_name) - add_node_fn(func_nid, f"{func_name}()", line) - add_edge_fn(file_nid, func_nid, "contains", line) - if callable_def_nids is not None: - callable_def_nids.add(func_nid) # `const f = () =>` is callable - if local_bound_names is not None: - local_bound_names[func_nid] = _js_local_bound_names(value, source) - body = value.child_by_field_name("body") - if body: - function_bodies.append((func_nid, body)) - arrow_found = True - elif value and value.type in ( - "object", "array", "as_expression", "call_expression", "new_expression", - ): - # Module-level const with literal/object/array/factory value - name_node = child.child_by_field_name("name") - if name_node: - const_name = _read_text(name_node, source) - line = child.start_point[0] + 1 - const_nid = _make_id(stem, const_name) - add_node_fn(const_nid, const_name, line) - add_edge_fn(file_nid, const_nid, "contains", line) - const_found = True - if arrow_found: - return True - if const_found: - return True - if require_found: - return True - return False +# ── Java extra walk for enum constants ─────────────────────────────────────── -# ── TS extra walk for namespace / module declarations ───────────────────────── - -def _ts_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: str, - nodes: list, edges: list, seen_ids: set, function_bodies: list, - parent_class_nid: str | None, add_node_fn, add_edge_fn, - walk_fn) -> bool: - """Emit a container node for a TS `namespace`/`module` declaration. - - `namespace Foo {}` parses as `internal_module` (with `name`/`body` fields); - `module Bar {}` and ambient `declare module "pkg" {}` parse as a named - `module` node that exposes no fields, so its name and body are found - positionally. Without this the container was never a node — its members were - still reached by the default recurse but lost their namespace context. The - members stay file-contained (parity with C#'s `_csharp_extra_walk`); the - namespace becomes a sibling marker node so it is queryable. Returns True if - handled. - - The guard requires `is_named` because the anonymous `module` keyword token - shares the `module` type string and would otherwise match here. - """ - if node.is_named and node.type in ("internal_module", "module"): - name_node = node.child_by_field_name("name") - if name_node is None: - for child in node.children: - if child.is_named and child.type in ( - "identifier", "nested_identifier", "string"): - name_node = child - break - body = node.child_by_field_name("body") - if body is None: - for child in node.children: - if child.type == "statement_block": - body = child - break - if name_node is not None: - ns_name = _read_text(name_node, source) - if name_node.type == "string": - ns_name = ns_name.strip("'\"`") - if ns_name: - ns_nid = _make_id(stem, ns_name) - line = node.start_point[0] + 1 - add_node_fn(ns_nid, ns_name, line) - add_edge_fn(file_nid, ns_nid, "contains", line) - if body is not None: - for child in body.children: - walk_fn(child, parent_class_nid) - return True - return False - - -# ── C# extra walk for namespace declarations ────────────────────────────────── - -def _csharp_namespace_name(node, source: bytes) -> str: - name_node = node.child_by_field_name("name") - if name_node is not None: - return _read_text(name_node, source).strip() - for child in node.children: - if child.type in ("identifier", "qualified_name"): - return _read_text(child, source).strip() - return "" - - -def _csharp_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: str, - nodes: list, edges: list, seen_ids: set, function_bodies: list, - parent_class_nid: str | None, add_node_fn, add_edge_fn, - walk_fn, namespace_stack: list[str], scope_stack: list[str]) -> bool: - """Handle namespace declarations for C#. Returns True if handled.""" - if node.type == "namespace_declaration": - ns_name = _csharp_namespace_name(node, source) - pushed = False - if ns_name: - namespace_stack.append(ns_name) - scope_stack.append(f"s{node.start_byte}") - pushed = True - ns_label = ".".join(namespace_stack) - ns_nid = _csharp_namespace_id(ns_label) - line = node.start_point[0] + 1 - add_node_fn(ns_nid, ns_label, line, node_type="namespace", metadata={"kind": "csharp_namespace"}) - add_edge_fn(file_nid, ns_nid, "contains", line) - body = node.child_by_field_name("body") - if body: - try: - for child in body.children: - walk_fn(child, parent_class_nid) - finally: - if pushed: - namespace_stack.pop() - scope_stack.pop() - elif pushed: - namespace_stack.pop() - scope_stack.pop() - return True - if node.type == "file_scoped_namespace_declaration": - ns_name = _csharp_namespace_name(node, source) - if ns_name: - namespace_stack.append(ns_name) - scope_stack.append(f"s{node.start_byte}") - ns_label = ".".join(namespace_stack) - ns_nid = _csharp_namespace_id(ns_label) - line = node.start_point[0] + 1 - add_node_fn(ns_nid, ns_label, line, node_type="namespace", metadata={"kind": "csharp_namespace"}) - add_edge_fn(file_nid, ns_nid, "contains", line) - return True - return False - - -# ── Swift extra walk for enum cases ────────────────────────────────────────── - -def _swift_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: str, - nodes: list, edges: list, seen_ids: set, function_bodies: list, - parent_class_nid: str | None, add_node_fn, add_edge_fn, - ensure_named_node_fn) -> bool: - """Handle enum_entry for Swift. Returns True if handled.""" - if node.type == "enum_entry" and parent_class_nid: - line = node.start_point[0] + 1 - for child in node.children: - if child.type == "simple_identifier": - case_name = _read_text(child, source) - case_nid = _make_id(parent_class_nid, case_name) - add_node_fn(case_nid, case_name, line) - add_edge_fn(parent_class_nid, case_nid, "case_of", line) - # Associated-value types nest as `enum_type_parameters -> user_type -> - # type_identifier` (a sibling of the case-name simple_identifier). The - # case-name loop above never descends into them, so `case started(Session)` - # used to drop the Event -> Session reference entirely. Mirror the Swift - # property/parameter emit style: collect the type refs and emit a - # `references` edge from the ENUM node to each collected type. - for child in node.children: - if child.type != "enum_type_parameters": - continue - for grand in child.children: - if not grand.is_named: - continue - refs: list[tuple[str, str]] = [] - _swift_collect_type_refs(grand, source, False, refs) - for ref_name, role in refs: - ctx = "generic_arg" if role == "generic_arg" else "type" - target_nid = ensure_named_node_fn(ref_name, line) - if target_nid != parent_class_nid: - add_edge_fn(parent_class_nid, target_nid, "references", - line, context=ctx) - return True - return False - - -# ── Language configs ────────────────────────────────────────────────────────── +# ── Language configs ────────────────────────────────────────────────────────── _PYTHON_CONFIG = LanguageConfig( ts_module="tree_sitter_python", @@ -3115,7 +892,7 @@ def _swift_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: s # older forks use `simple_identifier`. Accept both so the extractor # works across grammar generations. name_fallback_child_types=("simple_identifier", "identifier"), - body_fallback_child_types=("function_body", "class_body"), + body_fallback_child_types=("function_body", "class_body", "enum_class_body"), function_boundary_types=frozenset({"function_declaration"}), import_handler=_import_kotlin, ) @@ -3156,44 +933,6 @@ def _swift_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: s ) -def _resolve_lua_import_target(raw_module: str, str_path: str) -> str: - """Resolve a Lua require() module name to a node id. - - Lua module names use dots as path separators: `require("pkg.b")` looks for - `pkg/b.lua` (or `pkg/b/init.lua`) relative to a package root. We probe the - importing file's directory and walk upward looking for a matching file on - disk; if found, the returned id matches the file node id `_extract_generic` - assigns to that file (`_make_id(str(path))`), so the edge lands on a real - node. When nothing matches, fall back to `_make_id` of the full dotted - module name so cross-file resolution can still complete via the symbol - resolution pass instead of dropping the edge entirely (#1075). - """ - if not raw_module: - return "" - rel = raw_module.replace(".", "/") - try: - start_dir = Path(str_path).parent - except Exception: - start_dir = None - if start_dir is not None: - probe = start_dir - # Walk up a few levels so requires from nested files still resolve when - # the package root is above the importing file. - for _ in range(6): - for suffix in (".lua", ".luau"): - cand = probe / f"{rel}{suffix}" - if cand.is_file(): - return _make_id(str(cand)) - for suffix in (".lua", ".luau"): - cand = probe / rel / f"init{suffix}" - if cand.is_file(): - return _make_id(str(cand)) - if probe.parent == probe: - break - probe = probe.parent - return _make_id(raw_module) - - def _import_lua(node, source: bytes, file_nid: str, stem: str, edges: list, str_path: str, scope_stack: list[str] | None = None) -> None: """Extract require('module') from Lua variable_declaration nodes.""" text = _read_text(node, source) @@ -3263,31 +1002,6 @@ def _import_swift(node, source: bytes, file_nid: str, stem: str, edges: list, st return modules -def _read_csharp_type_name(node, source: bytes) -> tuple[str, bool, str] | None: - """Resolve a C# type name, whether it was qualified, and its qualifier prefix.""" - if node is None: - return None - if node.type in ("identifier", "predefined_type"): - return (_read_text(node, source), False, "") - if node.type == "qualified_name": - prefix, _, tail = _read_text(node, source).rpartition(".") - tail = tail.split("<", 1)[0] - return (tail, True, prefix) - if node.type == "generic_name": - name_node = node.child_by_field_name("name") - if name_node is not None: - qualified = name_node.type == "qualified_name" - prefix, _, tail = _read_text(name_node, source).rpartition(".") - return (tail, qualified, prefix if qualified else "") - for child in node.children: - if not child.is_named: - continue - result = _read_csharp_type_name(child, source) - if result: - return result - return None - - _SWIFT_CONFIG = LanguageConfig( ts_module="tree_sitter_swift", class_types=frozenset({"class_declaration", "protocol_declaration"}), @@ -3306,2546 +1020,142 @@ def _read_csharp_type_name(node, source: bytes) -> tuple[str, bool, str] | None: # ── Ruby local type inference (for member-call resolution) ───────────────────── -def _ruby_new_class_name(node, source: bytes) -> str | None: - """Return ``ClassName`` if ``node`` is a ``ClassName.new(...)`` call, else None. - - Only a bare capitalized constant receiver counts (``Processor.new``); - namespaced (``A::B.new``) and dynamic receivers are intentionally ignored so - the binding stays unambiguous. - """ - if node is None or node.type != "call": - return None - recv = node.child_by_field_name("receiver") - meth = node.child_by_field_name("method") - if recv is None or meth is None: - return None - if recv.type != "constant" or _read_text(meth, source) != "new": - return None - return _read_text(recv, source) - +# `Const = (...)` shapes that define a lightweight class named after the +# constant. tree-sitter parses each as an `assignment`, not a `class`, so the +# generic class branch never saw them (#1640). -def _ruby_local_class_bindings(body_node, source: bytes) -> dict[str, str | None]: - """Map ``local_var -> ClassName`` for ``var = ClassName.new`` within one Ruby - method body, not descending into nested method definitions. - 100%-confidence contract: a variable assigned more than once, or to anything - other than a single ``Constant.new``, maps to ``None`` (ambiguous) so callers - never resolve it. Only the certain single-binding case carries a type. - """ - bindings: dict[str, str | None] = {} - boundary = {"method", "singleton_method"} - - def visit(n) -> None: - for child in n.children: - if child.type in boundary: - continue # nested method has its own scope - if child.type == "assignment": - left = child.child_by_field_name("left") - right = child.child_by_field_name("right") - if left is not None and left.type == "identifier": - var = _read_text(left, source) - cls = _ruby_new_class_name(right, source) if right is not None else None - if cls is None: - # assigned to something we can't type: poison if it was typed - if var in bindings: - bindings[var] = None - elif var in bindings: - if bindings[var] != cls: - bindings[var] = None # reassigned to a different class - else: - bindings[var] = cls - visit(child) - - visit(body_node) - return bindings - - -def _ruby_const_last_name(node, source: bytes) -> str: - """Last constant of a ``constant`` or ``scope_resolution`` (``A::B::C`` -> ``C``).""" - if node is None: - return "" - if node.type == "constant": - return _read_text(node, source) - if node.type == "scope_resolution": - consts = [c for c in node.children if c.type == "constant"] - if consts: - return _read_text(consts[-1], source) - return "" +# ── Generic extractor ───────────────────────────────────────────────────────── -# `Const = (...)` shapes that define a lightweight class named after the -# constant. tree-sitter parses each as an `assignment`, not a `class`, so the -# generic class branch never saw them (#1640). -_RUBY_CLASS_FACTORIES = frozenset({("Struct", "new"), ("Class", "new"), ("Data", "define")}) - - -def _ruby_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: str, - nodes: list, edges: list, seen_ids: set, function_bodies: list, - parent_class_nid: str | None, add_node, add_edge, walk, - callable_def_nids: set) -> bool: - """Ruby: a constant assignment whose RHS is ``Struct.new(...)``, - ``Class.new(Super)`` or ``Data.define(...)`` defines a class named after the - constant (#1640). Synthesize the class node, attach block-defined methods via - ``method`` (by recursing the block with the new node as parent), and emit an - ``inherits`` edge for ``Class.new(Super)``. Returns True if handled. - """ - if node.type != "assignment": - return False - left = node.child_by_field_name("left") - right = node.child_by_field_name("right") - if left is None or right is None or left.type != "constant" or right.type != "call": - return False - recv = right.child_by_field_name("receiver") - meth = right.child_by_field_name("method") - if recv is None or meth is None or recv.type != "constant": - return False - if (_read_text(recv, source), _read_text(meth, source)) not in _RUBY_CLASS_FACTORIES: - return False +# ── Python rationale extraction ─────────────────────────────────────────────── - const_name = _read_text(left, source) - if not const_name: - return False - line = node.start_point[0] + 1 - class_nid = _make_id(stem, const_name) - add_node(class_nid, const_name, line) - callable_def_nids.add(class_nid) # a class is callable (its constructor) - # Mirror the generic class branch: containment always hangs off the file node. - add_edge(file_nid, class_nid, "contains", line) - - # `Class.new(Super)` — the first positional constant argument is the superclass. - if _read_text(recv, source) == "Class": - args = next((c for c in right.children if c.type == "argument_list"), None) - if args is not None: - for arg in args.children: - if arg.type in ("constant", "scope_resolution"): - base = _ruby_const_last_name(arg, source) - if base: - base_nid = _make_id(stem, base) - if base_nid not in seen_ids: - base_nid = _make_id(base) - if base_nid not in seen_ids: - nodes.append({ - "id": base_nid, "label": base, - "file_type": "code", "source_file": "", - "source_location": "", - }) - seen_ids.add(base_nid) - add_edge(class_nid, base_nid, "inherits", line) - break +_RATIONALE_PREFIXES = ("# NOTE:", "# IMPORTANT:", "# HACK:", "# WHY:", "# RATIONALE:", "# TODO:", "# FIXME:") - # Recurse the do/brace block so block-defined methods attach to the class. - # The block wraps its statements in a `body_statement` (like a class body); - # descend into it so the method handler sees parent_class_nid — otherwise the - # default recurse resets the parent to None and the method hangs off the file - # with a dot-less label. - block = next((c for c in right.children if c.type in ("do_block", "block")), None) - if block is not None: - body = next((c for c in block.children if c.type == "body_statement"), block) - for child in body.children: - walk(child, parent_class_nid=class_nid) - return True +def _is_autogenerated_python(source: bytes) -> bool: + """Return True if this Python file is auto-generated and its module docstring is noise. -# ── Generic extractor ───────────────────────────────────────────────────────── + Covers: Alembic/Flask-Migrate revisions, Django migrations, protobuf/gRPC/OpenAPI stubs. + Module docstrings in these files are change annotations or boilerplate, not rationale. + """ + head = source[:2048].decode("utf-8", errors="replace") + # Generic generated-file markers (protobuf, gRPC, OpenAPI codegen, etc.) + if any(m in head for m in ("DO NOT EDIT", "@generated", "Generated by the protocol buffer")): + return True + # Alembic / Flask-Migrate revision files + if (re.search(r"^revision\s*[:=]", head, re.MULTILINE) + and "def upgrade(" in head + and "down_revision" in head): + return True + # Django migrations + if "class Migration(migrations.Migration)" in head and "operations" in head: + return True + return False -def _extract_generic( - path: Path, config: LanguageConfig, *, source_override: bytes | None = None -) -> dict: - """Generic AST extractor driven by LanguageConfig. - ``source_override`` parses the given bytes instead of reading ``path``, while - still keying nodes/edges off ``path``. Lets container formats (e.g. Vue SFCs) - mask the wrapper and parse just the embedded `` close tag - pos = m.end() - if lang is None: - lang_m = _VUE_SCRIPT_LANG_RE.search(m.group(1)) - if lang_m: - lang = lang_m.group(1).lower() - out.append(_blank(src[pos:])) - return "".join(out), lang def extract_vue(path: Path) -> dict: @@ -6295,45 +1520,15 @@ def extract_vue(path: Path) -> dict: existing_ids = {n["id"] for n in result.get("nodes", [])} file_node_id = _make_id(str(path)) aliases = _load_tsconfig_aliases(path.parent) + base_url = _load_tsconfig_base_url(path.parent) for m in re.finditer(r"""import\(\s*['"]([^'"]+)['"]\s*\)""", src): raw = m.group(1) if not raw: continue - if raw.startswith("."): - resolved = Path(os.path.normpath(path.parent / raw)) - resolved = _resolve_js_module_path(resolved) - node_id = _make_id(str(resolved)) - stub_source_file = str(resolved) - else: - resolved_alias = _resolve_tsconfig_alias(raw, aliases) - if resolved_alias is not None: - resolved_alias = _resolve_js_module_path(resolved_alias) - node_id = _make_id(str(resolved_alias)) - stub_source_file = str(resolved_alias) - else: - module_name = raw.split("/")[-1] - if not module_name: - continue - node_id = _make_id(module_name) - stub_source_file = raw - if node_id in existing_ids: - result.setdefault("edges", []).append({ - "source": file_node_id, "target": node_id, - "relation": "dynamic_import", "confidence": "EXTRACTED", - "source_file": str(path), - }) - continue - result.setdefault("nodes", []).append({ - "id": node_id, "label": raw, - "file_type": "code", "source_file": stub_source_file, - "confidence": "EXTRACTED", - }) - result.setdefault("edges", []).append({ - "source": file_node_id, "target": node_id, - "relation": "dynamic_import", "confidence": "EXTRACTED", - "source_file": str(path), - }) - existing_ids.add(node_id) + _emit_rescued_import( + result, existing_ids, file_node_id, path, raw, + "dynamic_import", aliases, base_url, + ) except Exception: pass return result @@ -6480,7290 +1675,1340 @@ def extract_csharp(path: Path) -> dict: return _extract_generic(path, _CSHARP_CONFIG) -def extract_apex(path: Path) -> dict: - """Extract classes, interfaces, enums, methods, and Salesforce constructs from - Apex .cls and .trigger files using regex (no tree-sitter grammar on PyPI).""" - import re as _re - try: - source = path.read_text(encoding="utf-8", errors="replace") - except OSError: - return {"nodes": [], "edges": []} - - str_path = str(path) - stem = _file_stem(path) - file_nid = _make_id(str_path) - - nodes: list[dict] = [] - edges: list[dict] = [] - seen_ids: set[str] = set() - - def add_node(nid: str, label: str, line: int) -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": label, - "file_type": "code", - "source_file": str_path, - "source_location": f"L{line}", - }) - - def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED") -> None: - edges.append({ - "source": src, - "target": tgt, - "relation": relation, - "confidence": confidence, - "source_file": str_path, - "source_location": f"L{line}", - "weight": 1.0, - }) - - add_node(file_nid, path.name, 1) +def extract_kotlin(path: Path) -> dict: + """Extract classes, objects, functions, and imports from a .kt/.kts file.""" + return _extract_generic(path, _KOTLIN_CONFIG) - lines = source.splitlines() - _ACCESS = r"(?:public|private|protected|global|webService)?" - _SHARING = r"(?:\s+(?:with|without|inherited)\s+sharing)?" - _MOD = r"(?:\s+(?:abstract|virtual|override|static|final|transient|testMethod))?" - _ANNOTATION = r"(?:\s*@\w+(?:\s*\([^)]*\))?\s*)*" +def extract_scala(path: Path) -> dict: + """Extract classes, objects, functions, and imports from a .scala file.""" + return _extract_generic(path, _SCALA_CONFIG) - cls_re = _re.compile( - rf"^{_ANNOTATION}\s*{_ACCESS}{_SHARING}{_MOD}\s*class\s+(\w+)" - rf"(?:\s+extends\s+(\w+))?(?:\s+implements\s+([\w,\s]+))?\s*\{{?", - _re.IGNORECASE, - ) - iface_re = _re.compile( - rf"^{_ANNOTATION}\s*{_ACCESS}{_SHARING}{_MOD}\s*interface\s+(\w+)" - rf"(?:\s+extends\s+([\w,\s]+))?\s*\{{?", - _re.IGNORECASE, - ) - enum_re = _re.compile( - rf"^{_ANNOTATION}\s*{_ACCESS}{_SHARING}{_MOD}\s*enum\s+(\w+)\s*\{{?", - _re.IGNORECASE, - ) - trigger_re = _re.compile( - r"^\s*trigger\s+(\w+)\s+on\s+(\w+)\s*\(", - _re.IGNORECASE, - ) - method_re = _re.compile( - rf"^{_ANNOTATION}\s*{_ACCESS}{_MOD}\s*(?:static\s+)?[\w<>\[\]]+\s+(\w+)\s*\([^)]*\)\s*(?:throws\s+\w+\s*)?\{{?", - _re.IGNORECASE, - ) - annotation_re = _re.compile(r"@(\w+)", _re.IGNORECASE) - soql_re = _re.compile(r"\[\s*SELECT\b[^\]]+FROM\s+(\w+)", _re.IGNORECASE) - dml_re = _re.compile(r"\b(insert|update|delete|upsert|merge|undelete)\s+\w", _re.IGNORECASE) - - _CONTROL_FLOW = frozenset({ - "if", "else", "for", "while", "do", "switch", "try", "catch", - "finally", "return", "throw", "new", "void", "null", - "true", "false", "this", "super", "class", "interface", "enum", - "trigger", "on", - }) - current_class_nid: str | None = None - pending_annotations: list[str] = [] +def extract_php(path: Path) -> dict: + """Extract classes, functions, methods, namespace uses, and calls from a .php file.""" + return _extract_generic(path, _PHP_CONFIG) - for lineno, line_text in enumerate(lines, start=1): - stripped = line_text.strip() - if stripped.startswith("@"): - for m in annotation_re.finditer(stripped): - pending_annotations.append(m.group(1).lower()) - continue +# One level of balanced parens (e.g. `Foo #(Bar #(int))`) — bounded so malformed +# input cannot trigger pathological backtracking. - tm = trigger_re.match(stripped) - if tm: - trig_name, sobject = tm.group(1), tm.group(2) - trig_nid = _make_id(stem, trig_name) - add_node(trig_nid, trig_name, lineno) - add_edge(file_nid, trig_nid, "contains", lineno) - sob_nid = _make_id(sobject) - if sob_nid not in seen_ids: - add_node(sob_nid, sobject, lineno) - add_edge(trig_nid, sob_nid, "uses", lineno, confidence="INFERRED") - current_class_nid = trig_nid - pending_annotations = [] - continue - cm = cls_re.match(stripped) - if cm: - class_name = cm.group(1) - if class_name.lower() in _CONTROL_FLOW: - pending_annotations = [] - continue - class_nid = _make_id(stem, class_name) - add_node(class_nid, class_name, lineno) - add_edge(file_nid, class_nid, "contains", lineno) - if cm.group(2): - base = cm.group(2).strip() - base_nid = _make_id(stem, base) - if base_nid not in seen_ids: - base_nid = _make_id(base) - if base_nid not in seen_ids: - add_node(base_nid, base, lineno) - add_edge(class_nid, base_nid, "extends", lineno, confidence="INFERRED") - if cm.group(3): - for iface in cm.group(3).split(","): - iface = iface.strip() - if iface: - iface_nid = _make_id(stem, iface) - if iface_nid not in seen_ids: - iface_nid = _make_id(iface) - if iface_nid not in seen_ids: - add_node(iface_nid, iface, lineno) - add_edge(class_nid, iface_nid, "implements", lineno, confidence="INFERRED") - current_class_nid = class_nid - pending_annotations = [] - continue +def extract_lua(path: Path) -> dict: + """Extract functions, methods, require() imports, and calls from a .lua file.""" + return _extract_generic(path, _LUA_CONFIG) - im = iface_re.match(stripped) - if im: - iface_name = im.group(1) - if iface_name.lower() in _CONTROL_FLOW: - pending_annotations = [] - continue - iface_nid = _make_id(stem, iface_name) - add_node(iface_nid, iface_name, lineno) - add_edge(file_nid if current_class_nid is None else current_class_nid, - iface_nid, "contains", lineno) - if im.group(2): - for parent in im.group(2).split(","): - parent = parent.strip() - if parent: - parent_nid = _make_id(stem, parent) - if parent_nid not in seen_ids: - parent_nid = _make_id(parent) - if parent_nid not in seen_ids: - add_node(parent_nid, parent, lineno) - add_edge(iface_nid, parent_nid, "extends", lineno, confidence="INFERRED") - pending_annotations = [] - continue - em = enum_re.match(stripped) - if em: - enum_name = em.group(1) - if enum_name.lower() in _CONTROL_FLOW: - pending_annotations = [] - continue - enum_nid = _make_id(stem, enum_name) - add_node(enum_nid, enum_name, lineno) - add_edge(file_nid if current_class_nid is None else current_class_nid, - enum_nid, "contains", lineno) - pending_annotations = [] - continue +def extract_swift(path: Path) -> dict: + """Extract classes, structs, protocols, functions, imports, and calls from a .swift file.""" + return _extract_generic(path, _SWIFT_CONFIG) - if current_class_nid is not None: - mm = method_re.match(stripped) - if mm: - method_name = mm.group(1) - if method_name.lower() not in _CONTROL_FLOW: - method_nid = _make_id(current_class_nid, method_name) - method_label = f".{method_name}()" - add_node(method_nid, method_label, lineno) - add_edge(current_class_nid, method_nid, "method", lineno) - if "auraenabled" in pending_annotations or "invocablemethod" in pending_annotations: - add_edge(file_nid, method_nid, "contains", lineno, confidence="INFERRED") - pending_annotations = [] - continue - pending_annotations = [] +# ── Julia extractor (custom walk) ──────────────────────────────────────────── - for sm in soql_re.finditer(line_text): - sobject = sm.group(1) - sob_nid = _make_id(sobject) - if sob_nid not in seen_ids: - add_node(sob_nid, sobject, lineno) - src = current_class_nid or file_nid - add_edge(src, sob_nid, "uses", lineno, confidence="INFERRED") - for dm in dml_re.finditer(line_text): - dml_op = dm.group(1).lower() - dml_nid = _make_id(f"dml_{dml_op}") - if dml_nid not in seen_ids: - add_node(dml_nid, dml_op, lineno) - src = current_class_nid or file_nid - add_edge(src, dml_nid, "uses", lineno, confidence="INFERRED") +# ── Go extractor (custom walk) ──────────────────────────────────────────────── - return {"nodes": nodes, "edges": edges} +# ── Rust extractor (custom walk) ────────────────────────────────────────────── -def extract_kotlin(path: Path) -> dict: - """Extract classes, objects, functions, and imports from a .kt/.kts file.""" - return _extract_generic(path, _KOTLIN_CONFIG) +# Common Rust trait/stdlib method names that appear in virtually every codebase. +# Resolving these cross-file produces spurious INFERRED edges across crate +# boundaries (issue #908) — skip them from the unresolved-call queue entirely. -def extract_scala(path: Path) -> dict: - """Extract classes, objects, functions, and imports from a .scala file.""" - return _extract_generic(path, _SCALA_CONFIG) +# ── Zig ─────────────────────────────────────────────────────────────────────── -def extract_php(path: Path) -> dict: - """Extract classes, functions, methods, namespace uses, and calls from a .php file.""" - return _extract_generic(path, _PHP_CONFIG) +# ── PowerShell ──────────────────────────────────────────────────────────────── +# ── PowerShell manifest (.psd1) ────────────────────────────────────────────── +# Keys in a .psd1 whose values are module names/paths we treat as imports. -def extract_dart(path: Path) -> dict: - """Extract classes, mixins, functions, imports, generic calls, and annotations from a .dart file using regex.""" - try: - src = path.read_text(encoding="utf-8", errors="replace") - except OSError: - return {"error": f"cannot read {path}"} - - # Remove inline and multi-line comments while leaving string literals untouched to prevent stripping URLs/paths inside strings - comment_string_pattern = re.compile( - r'"""(?:\\.|[\s\S])*?"""' - r"|'''(?:\\.|[\s\S])*?'''" - r'|"(?:\\.|[^"\\])*"' - r"|'(?:\\.|[^'\\])*'" - r"|/\*[\s\S]*?\*/" - r"|//[^\n]*" - ) - def _comment_replace(match: re.Match) -> str: - token = match.group(0) - if token.startswith("/"): - return "" - return token - src_clean = comment_string_pattern.sub(_comment_replace, src) - stem = _file_stem(path) - file_nid = _make_id(str(path)) +# ── Cross-file import resolution ────────────────────────────────────────────── - # Check if this is a part-of file and redirect to parent - part_of_match = re.search(r"^\s*part\s+of\s+['\"]([^'\"]+)['\"]", src_clean, re.MULTILINE) - is_part = False - if part_of_match: - parent_ref = part_of_match.group(1) - if parent_ref.endswith(".dart"): - try: - parent_path = (path.parent / parent_ref).resolve() - if parent_path.exists(): - stem = _file_stem(parent_path) - file_nid = _make_id(str(parent_path)) - is_part = True - except Exception: - pass - nodes = [] - if not is_part: - nodes.append({"id": file_nid, "label": path.name, "file_type": "code", - "source_file": str(path), "source_location": None}) - edges = [] - defined: set[str] = set() - - def add_node(nid: str, label: str, ftype: str = "code", source_file: str | None = str(path)) -> None: - if nid not in defined: - nodes.append({"id": nid, "label": label, "file_type": ftype, - "source_file": source_file, "source_location": None}) - defined.add(nid) - - def add_edge(src_id: str, tgt_id: str, relation: str, weight: float = 1.0, context: str | None = None) -> None: - edge = {"source": src_id, "target": tgt_id, "relation": relation, - "confidence": "EXTRACTED", "confidence_score": 1.0, - "source_file": str(path), "source_location": None, "weight": weight} - if context: - edge["context"] = context - edges.append(edge) +def _canonicalize_csharp_namespace_nodes(all_nodes: list[dict], all_edges: list[dict]) -> None: + """Collapse duplicate C# namespace node entries to one canonical node per label.""" + by_label: dict[str, list[dict]] = {} + for node in all_nodes: + if node.get("type") != "namespace": + continue + label = node.get("label") + if isinstance(label, str): + by_label.setdefault(label, []).append(node) - def _split_types(text: str) -> list[str]: - parts = [] - current = [] - depth = 0 - for char in text: - if char == "<": - depth += 1 - current.append(char) - elif char == ">": - depth -= 1 - current.append(char) - elif char == "," and depth == 0: - parts.append("".join(current).strip()) - current = [] - else: - current.append(char) - if current: - parts.append("".join(current).strip()) - return [p for p in parts if p] - - def _find_matching_brace(text: str, start_pos: int) -> int: - brace_count = 0 - in_double_quote = False - in_single_quote = False - escape = False - - first_brace = text.find("{", start_pos) - if first_brace == -1: - return len(text) - - brace_count = 1 - i = first_brace + 1 - n = len(text) - while i < n: - char = text[i] - if escape: - escape = False - i += 1 - continue - if char == "\\": - escape = True - i += 1 - continue - if text[i:i+3] == '"""' and not in_single_quote: - i += 3 - end = text.find('"""', i) - i = end + 3 if end != -1 else n - continue - if text[i:i+3] == "'''" and not in_double_quote: - i += 3 - end = text.find("'''", i) - i = end + 3 if end != -1 else n - continue - if char == '"' and not in_single_quote: - in_double_quote = not in_double_quote - elif char == "'" and not in_double_quote: - in_single_quote = not in_single_quote - elif not in_double_quote and not in_single_quote: - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - return i + 1 - i += 1 - return len(text) - - # 1. Classes, mixins, and enums declarations (with inheritance, mixins, interfaces, and generics) - # Supports multiple combined modifiers (e.g., abstract base class, mixin class) without capturing "class" as a name - class_pattern = r"^\s*(?:(?:abstract|sealed|base|interface|final|mixin)\s+)*(?:class|mixin|enum|extension\s+type)\s+(\w+)" - for m in re.finditer(class_pattern, src_clean, re.MULTILINE): - class_name = m.group(1) - class_nid = _make_id(stem, class_name) - add_node(class_nid, class_name) - add_edge(file_nid, class_nid, "defines") - - # Manually parse extends/on, with, and implements in header to handle nested generics brackets balanced - start_idx = m.end() - rest = src_clean[start_idx : start_idx + 500] - - # Skip class generic parameters - if rest.lstrip().startswith("<"): - offset = rest.find("<") - depth = 1 - i = offset + 1 - while i < len(rest) and depth > 0: - if rest[i] == "<": depth += 1 - elif rest[i] == ">": depth -= 1 - i += 1 - rest = rest[i:] - - # Skip primary constructor (e.g. extension type MyExt(int id)) - if rest.lstrip().startswith("("): - offset = rest.find("(") - depth = 1 - i = offset + 1 - while i < len(rest) and depth > 0: - if rest[i] == "(": depth += 1 - elif rest[i] == ")": depth -= 1 - i += 1 - rest = rest[i:] - - header_end = rest.find("{") - if header_end == -1: - header_end = rest.find(";") - if header_end == -1: - header_end = len(rest) - header = rest[:header_end] - - base_class = None - generics = None - mixins_list = [] - interfaces_list = [] - - # Parse extends or on - extends_m = re.search(r"^\s*(?:extends|on)\s+([a-zA-Z0-9_.]+)", header) - if extends_m: - base_class = extends_m.group(1) - rest_header = header[extends_m.end():] - if rest_header.strip().startswith("<"): - start_idx = rest_header.find("<") - depth = 1 - i = start_idx + 1 - while i < len(rest_header) and depth > 0: - if rest_header[i] == "<": - depth += 1 - elif rest_header[i] == ">": - depth -= 1 - if depth == 0: - generics = rest_header[start_idx + 1 : i] - break - i += 1 - if generics is not None: - header = rest_header[i + 1:] - else: - header = rest_header - else: - header = rest_header - - # Parse with - with_m = re.search(r"^\s*with\s+", header) - if with_m: - rest_header = header[with_m.end():] - impl_idx = rest_header.find("implements") - if impl_idx != -1: - mixins_str = rest_header[:impl_idx] - header = rest_header[impl_idx:] - else: - mixins_str = rest_header - header = "" - mixins_list = _split_types(mixins_str) - - # Parse implements - impl_m = re.search(r"^\s*implements\s+", header) - if impl_m: - interfaces_list = _split_types(header[impl_m.end():]) - - # Map extends inheritance relation - if base_class: - base_nid = _make_id(base_class) - add_node(base_nid, base_class, source_file=None) - add_edge(class_nid, base_nid, "inherits") - - # Map generic type arguments (e.g. MyBloc extends Bloc) - if generics: - for gen in _split_types(generics): - gen_clean = gen.split("<")[0].strip() - if gen_clean not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "void"}: - gen_nid = _make_id(gen_clean) - add_node(gen_nid, gen_clean, source_file=None) - add_edge(class_nid, gen_nid, "references") - - # Map mixins - for mixin in mixins_list: - mixin_clean = mixin.split("<")[0].strip() - mixin_nid = _make_id(mixin_clean) - add_node(mixin_nid, mixin_clean, source_file=None) - add_edge(class_nid, mixin_nid, "mixes_in") - - # Map interfaces - for interface in interfaces_list: - interface_clean = interface.split("<")[0].strip() - interface_nid = _make_id(interface_clean) - add_node(interface_nid, interface_clean, source_file=None) - add_edge(class_nid, interface_nid, "implements") - - # Extract class body for precise framework dependencies and event handling - start_idx = m.start() - brace_pos = src_clean.find("{", start_idx) - semi_pos = src_clean.find(";", start_idx) - - has_body = brace_pos != -1 - if has_body and semi_pos != -1 and semi_pos < brace_pos: - has_body = False - - if has_body: - end_pos = _find_matching_brace(src_clean, start_idx) - class_body = src_clean[brace_pos:end_pos] - - # Bloc event registration: on() - for em in re.finditer(r"\bon<(\w+)>\s*\(", class_body): - event_name = em.group(1) - event_nid = _make_id(event_name) - add_node(event_nid, event_name, source_file=None) - add_edge(class_nid, event_nid, "calls", context="bloc_event") - - # Bloc state emissions: emit(MyState) or yield MyState - for sm in re.finditer(r"\b(?:emit|yield)\s*\(?\s*(?:const\s+)?([A-Z]\w*)\b", class_body): - state_name = sm.group(1) - if state_name not in {"String", "List", "Map", "Set", "Future", "Stream", "Object"}: - state_nid = _make_id(state_name) - add_node(state_nid, state_name, source_file=None) - add_edge(class_nid, state_nid, "calls", context="emit_state") - - # Bloc event additions: widget.add(MyEvent()) or bloc.add(MyEvent()) - for am in re.finditer(r"\b(?:\w*[Bb]loc\w*|context\.read<\w+>\(\))\.add\(\s*(?:const\s+)?([A-Z]\w*)\b", class_body): - event_name = am.group(1) - if event_name not in {"String", "List", "Map", "Set", "Future", "Stream", "Object"}: - event_nid = _make_id(event_name) - add_node(event_nid, event_name, source_file=None) - add_edge(class_nid, event_nid, "calls", context="bloc_add_event") - - # Riverpod provider references: ref.watch(provider) - for rm in re.finditer(r"\bref\.(?:watch|read|listen)\s*\(\s*(\w+)\b", class_body): - provider_name = rm.group(1) - provider_nid = _make_id(provider_name) - add_node(provider_nid, provider_name, source_file=None) - add_edge(class_nid, provider_nid, "references", context="riverpod_reference") - - # Widget to Bloc references: BlocBuilder - for bm in re.finditer(r"\bBloc(?:Builder|Listener|Consumer|Provider|Selector)\s*<\s*([a-zA-Z0-9_]+)\b", class_body): - bloc_name = bm.group(1) - if bloc_name not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "void"}: - bloc_nid = _make_id(bloc_name) - add_node(bloc_nid, bloc_name, source_file=None) - add_edge(class_nid, bloc_nid, "references", context="bloc_widget_binding") - - # context.read() or BlocProvider.of(context) - for lm in re.finditer(r"\b(?:read|watch|select|of)\s*<([a-zA-Z0-9_]+)>", class_body): - bloc_name = lm.group(1) - if bloc_name not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "void"}: - bloc_nid = _make_id(bloc_name) - add_node(bloc_nid, bloc_name, source_file=None) - add_edge(class_nid, bloc_nid, "references", context="bloc_lookup") - - # 2. Annotations mapping (class, mixin, enum, or function level annotations) - # Support: @riverpod, @Riverpod(...), @injectable, @singleton, @RoutePage(), @HiveType(typeId: 0), @RestApi() - # Matches `@annotation` and links it to the next class/mixin/enum/function declaration in the file - annotation_pattern = r"@(\w+)(?:\([^)]*\))?" - for am in re.finditer(annotation_pattern, src_clean): - annotation_name = am.group(1) - if annotation_name in {"override", "deprecated", "required", "protected", "mustCallSuper"}: + remap: dict[str, str] = {} + drop_node_ids: set[int] = set() + for group in by_label.values(): + if len(group) < 2: continue - annotation_pos = am.end() - intervening_text = src_clean[annotation_pos : annotation_pos + 300] + canonical = sorted( + group, + key=lambda node: ( + str(node.get("source_file") or ""), + str(node.get("source_location") or ""), + str(node.get("id") or ""), + ), + )[0] + canonical_id = canonical.get("id") + for node in group: + if node is canonical: + continue + drop_node_ids.add(id(node)) + dup_id = node.get("id") + if isinstance(dup_id, str) and isinstance(canonical_id, str): + remap[dup_id] = canonical_id - class_m = re.search(r"^\s*(?:(?:abstract|sealed|base|interface|final|mixin)\s+)*(?:class|mixin|enum|extension\s+type)\s+(\w+)", intervening_text, re.MULTILINE) - func_m = re.search(r"^\s*(?:factory\s+|static\s+|async\s+|external\s+|abstract\s+)?(?:\([^)]+\)|[a-zA-Z0-9_<>,.?]+)(?:\s+[a-zA-Z0-9_<>,.?]+){0,3}\s+(\w+)\s*\(", intervening_text, re.MULTILINE) + if remap: + for edge in all_edges: + if edge.get("source") in remap: + edge["source"] = remap[str(edge["source"])] + if edge.get("target") in remap: + edge["target"] = remap[str(edge["target"])] - target_nid = None - target_name = None - target_type = None + if drop_node_ids: + all_nodes[:] = [node for node in all_nodes if id(node) not in drop_node_ids] - if class_m and func_m: - if class_m.start() < func_m.start(): - target_name = class_m.group(1) - target_type = "class" - target_nid = _make_id(stem, target_name) - else: - target_name = func_m.group(1) - target_type = "function" - target_nid = _make_id(stem, target_name) - elif class_m: - target_name = class_m.group(1) - target_type = "class" - target_nid = _make_id(stem, target_name) - elif func_m: - target_name = func_m.group(1) - target_type = "function" - target_nid = _make_id(stem, target_name) - - if target_nid and target_name: - actual_intervening = intervening_text[:min(class_m.start() if class_m else 300, func_m.start() if func_m else 300)] - if ";" not in actual_intervening and "}" not in actual_intervening and "{" not in actual_intervening: - annotation_nid = _make_id("annotation", annotation_name.lower()) - add_node(annotation_nid, f"@{annotation_name}", ftype="concept", source_file=None) - add_edge(target_nid, annotation_nid, "configures") - - # Riverpod specific provider generation mapping (supports camelCase class and functional providers) - if annotation_name.lower() == "riverpod": - if target_type == "class": - provider_name = target_name[0].lower() + target_name[1:] + "Provider" if len(target_name) > 1 else target_name.lower() + "Provider" - else: - provider_name = target_name + "Provider" - provider_nid = _make_id(provider_name) - add_node(provider_nid, provider_name, ftype="concept", source_file=str(path)) - add_edge(target_nid, provider_nid, "defines", context="riverpod_provider") - - # 2.5 Typedefs (Type Aliases) - typedef_pattern = r"^\s*typedef\s+(\w+)\s*(?:<[^>]+>)?\s*=\s*([a-zA-Z0-9_<>,.?\s]+);" - for m in re.finditer(typedef_pattern, src_clean, re.MULTILINE): - typedef_name = m.group(1) - target_type = m.group(2).split("<")[0].split(".")[-1].strip() - if target_type not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "List", "Map", "Set", "void", "Function"}: - typedef_nid = _make_id(stem, typedef_name) - add_node(typedef_nid, typedef_name) - add_edge(file_nid, typedef_nid, "defines") - target_nid = _make_id(target_type) - add_node(target_nid, target_type, source_file=None) - add_edge(typedef_nid, target_nid, "references", context="typedef") - - # 3. Extensions (extension MyExt on MyClass) - ext_pattern = r"^\s{0,4}extension\s+(\w+)?(?:<[^>]+>)?\s+on\s+(\w+)" - for m in re.finditer(ext_pattern, src_clean, re.MULTILINE): - ext_name = m.group(1) or f"{stem}_anonymous_extension" - target_class = m.group(2) - - ext_nid = _make_id(stem, ext_name) - label = m.group(1) or f"Extension on {target_class}" - add_node(ext_nid, label) - add_edge(file_nid, ext_nid, "defines") - - target_nid = _make_id(target_class) - add_node(target_nid, target_class, source_file=None) - add_edge(ext_nid, target_nid, "extends") - - # 4. Top-level and class-level variable declarations (generic variables, records, late, and destructuring) - # Restrict indentation to 0-2 spaces to avoid matching local variables inside functions or switch expressions - var_pattern = r"^\s{0,2}(?:late\s+)?(?:(?:final|const|var)\s+)?(?:\([^)]+\)\s+|([a-zA-Z0-9_<>,.?]+(?:\s+[a-zA-Z0-9_<>,.?]+){0,3})\s+)?(?:(\w+)|(?:\w+\s*)?\(([^)]+)\))\s*(?:=|$|;)" - for m in re.finditer(var_pattern, src_clean, re.MULTILINE): - var_type = m.group(1) - single_name = m.group(2) - destructured_names = m.group(3) - - if not re.match(r"^\s*(?:late|final|const|var)\b", m.group(0)) and not var_type: - continue - if single_name: - if single_name not in {"if", "for", "while", "switch", "catch", "return"}: - var_nid = _make_id(stem, single_name) - add_node(var_nid, single_name) - add_edge(file_nid, var_nid, "defines") - - if var_type and var_type not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "List", "Map", "Set", "void"}: - clean_type = var_type.split("<")[0].split(".")[-1].strip() - type_nid = _make_id(clean_type) - add_node(type_nid, clean_type, source_file=None) - add_edge(file_nid, type_nid, "references", context="variable_type") - elif destructured_names: - for name in [n.strip() for n in destructured_names.split(",") if n.strip()]: - if ":" in name: - name = name.split(":")[-1].strip() - if re.match(r"^[a-zA-Z_]\w*$", name) and not re.match(r"^[A-Z]", name): - if name not in {"if", "for", "while", "switch", "catch", "return"}: - var_nid = _make_id(stem, name) - add_node(var_nid, name) - add_edge(file_nid, var_nid, "defines") - - # 5. Top-level and member functions/methods (supports typed/generic/record return types and Riverpod/Bloc references) - # Restrict indentation to 0-2 spaces to avoid matching nested local functions or methods inside multiline switch statements - method_pattern = r"^\s{0,2}(?:factory\s+|static\s+|async\s+|external\s+|abstract\s+)?(?:\([^)]+\)|[a-zA-Z0-9_<>,.?]+)(?:\s+[a-zA-Z0-9_<>,.?]+){0,3}\s+(\w+(?:\.\w+)?)\s*\(" - for m in re.finditer(method_pattern, src_clean, re.MULTILINE): - raw_name = m.group(1) - name = raw_name.split(".")[-1] - if name in {"if", "for", "while", "switch", "catch", "return", "void", "dynamic", "final", "const", "get", "set"}: - continue - if re.match(r"^[A-Z]", name): - continue - nid = _make_id(stem, name) - add_node(nid, name) - add_edge(file_nid, nid, "defines") - - # Get function body using matching brace to extract Riverpod reference patterns - start_idx = m.start() - brace_pos = src_clean.find("{", start_idx) - semi_pos = src_clean.find(";", start_idx) - arrow_pos = src_clean.find("=>", start_idx) - - has_body = brace_pos != -1 - if has_body and semi_pos != -1 and semi_pos < brace_pos: - has_body = False - if has_body and arrow_pos != -1 and arrow_pos < brace_pos: - has_body = False - - if has_body: - end_pos = _find_matching_brace(src_clean, start_idx) - func_body = src_clean[brace_pos:end_pos] - - # Extract Riverpod provider references: ref.watch(provider) - for rm in re.finditer(r"\bref\.(?:watch|read|listen)\s*\(\s*(\w+)\b", func_body): - provider_name = rm.group(1) - provider_nid = _make_id(provider_name) - add_node(provider_nid, provider_name, source_file=None) - add_edge(nid, provider_nid, "references", context="riverpod_reference") - - # Extract Bloc event additions: widget.add(MyEvent()) or bloc.add(MyEvent()) - for am in re.finditer(r"\b(?:\w*[Bb]loc\w*|context\.read<\w+>\(\))\.add\(\s*(?:const\s+)?([A-Z]\w*)\b", func_body): - event_name = am.group(1) - if event_name not in {"String", "List", "Map", "Set", "Future", "Stream", "Object"}: - event_nid = _make_id(event_name) - add_node(event_nid, event_name, source_file=None) - add_edge(nid, event_nid, "calls", context="bloc_add_event") - - # context.read() or BlocProvider.of(context) - for lm in re.finditer(r"\b(?:read|watch|select|of)\s*<([a-zA-Z0-9_]+)>", func_body): - bloc_name = lm.group(1) - if bloc_name not in {"String", "int", "double", "bool", "num", "dynamic", "Object", "void"}: - bloc_nid = _make_id(bloc_name) - add_node(bloc_nid, bloc_name, source_file=None) - add_edge(nid, bloc_nid, "references", context="bloc_lookup") - - # Universal Navigation Patters (GoRouter, AutoRoute, Navigator) - for nm in re.finditer(r"\b(?:go|push|goNamed|pushNamed|replace|replaceNamed)\s*\(\s*(?:context\s*,\s*)?['\"]([a-zA-Z0-9_/?=&%-]+)['\"]", func_body): - route_path = nm.group(1) - route_nid = _make_id("route", route_path.replace("/", "_").replace("?", "_").replace("=", "_").replace("&", "_")) - add_node(route_nid, f"Route {route_path}", ftype="concept", source_file=None) - add_edge(nid, route_nid, "navigates", context="route_path") - - for cm in re.finditer(r"\b(?:go|push|goNamed|pushNamed|replace|replaceNamed)\s*\(\s*(?:context\s*,\s*)?([A-Z][a-zA-Z0-9_]*\.[a-zA-Z0-9_]+)", func_body): - route_const = cm.group(1) - route_nid = _make_id("route", route_const.replace(".", "_")) - add_node(route_nid, route_const, ftype="concept", source_file=None) - add_edge(nid, route_nid, "navigates", context="route_const") - - for om in re.finditer(r"\b(?:push|replace)\s*\(\s*(?:context\s*,\s*)?.*?\b([A-Z]\w*(?:Route|Screen|Page))\b", func_body): - route_class = om.group(1) - route_nid = _make_id(route_class) - add_node(route_nid, route_class, source_file=None) - add_edge(nid, route_nid, "navigates", context="route_object") - - # 6. Imports and Exports - for m in re.finditer(r"""^\s*import\s+['"]([^'"]+)['"]""", src_clean, re.MULTILINE): - pkg = m.group(1) - tgt_nid = _make_id(pkg) - add_node(tgt_nid, pkg, source_file=None) - add_edge(file_nid, tgt_nid, "imports") - - for m in re.finditer(r"""^\s*export\s+['"]([^'"]+)['"]""", src_clean, re.MULTILINE): - pkg = m.group(1) - tgt_nid = _make_id(pkg) - add_node(tgt_nid, pkg, source_file=None) - add_edge(file_nid, tgt_nid, "exports") - - # 7. Generic Invocations / Type Lookups (Universal Dependency Lookup) - # Matches any method call with type parameters: methodName() or object.methodName() - # Automatically extracts GetIt, Injectable, Riverpod, Provider, BlocProvider, and InheritedWidget type lookups! - generic_call_pattern = r"\b\w+<([a-zA-Z0-9_.]+(?:<[a-zA-Z0-9_.,\s<>]+>)?)\s*>\s*\(" - type_blacklist = {"String", "int", "double", "bool", "num", "dynamic", "Object", "List", "Map", "Set", "Future", "Stream", "void"} - for m in re.finditer(generic_call_pattern, src_clean): - type_name = m.group(1).split(".")[-1].strip() - clean_name = type_name.split("<")[0].strip() - if clean_name not in type_blacklist: - target_nid = _make_id(clean_name) - add_node(target_nid, clean_name, source_file=None) - add_edge(file_nid, target_nid, "references", context="type_lookup") +# Languages whose identifiers are case-insensitive, so cross-file name resolution +# may fold case. Everywhere else, case is semantic (`Path` the class vs `PATH` the +# env var are distinct) and folding manufactures false edges / super-hubs (#1581). +_CASE_INSENSITIVE_EXTS = frozenset({ + ".php", ".phtml", ".php3", ".php4", ".php5", ".php7", ".phps", # PHP fns/classes + ".sql", # SQL identifiers + ".nim", ".nims", ".nimble", # Nim (style-insensitive) +}) - return {"nodes": nodes, "edges": edges} +def _lang_is_case_insensitive(source_file: object) -> bool: + """True when the file's language resolves identifiers case-insensitively (#1581).""" + if not source_file: + return False + return Path(str(source_file)).suffix.lower() in _CASE_INSENSITIVE_EXTS -def _sv_first_identifier(node, source: bytes) -> str | None: - """First `simple_identifier` under node in pre-order, or None. - tree-sitter-verilog 1.0.3 nests declaration names a few levels deep instead - of exposing a `name` field. Scope the search to the right child node (e.g. - `function_identifier`) or this returns the return-type instead of the name. - """ - if node is None: - return None - for child in node.children: - if child.type == "simple_identifier": - return _read_text(child, source) - found = _sv_first_identifier(child, source) - if found: - return found - return None +# Language interop families for cross-file call resolution. A call in one language +# can never bind by name to a definition in another family — a TSX component does +# not invoke a Kotlin method, and a Python function does not invoke a Java one. +# Families are grouped by REAL interop so legitimate cross-language resolution +# keeps working: Kotlin/Java/Scala/Groovy share the JVM, C/C++/Objective-C/CUDA +# share headers and symbols (Swift bridges to Objective-C), and JS/TS variants +# (plus Vue/Svelte/Astro SFC script blocks) compile into one module graph. +# Extensions absent from this map (docs, configs, unknown languages) resolve to +# no family and are never filtered — same permissive default as before. +_LANG_FAMILY_BY_EXT: dict[str, str] = { + # JS/TS module graph (SFCs embed JS/TS) + ".js": "jsts", ".jsx": "jsts", ".mjs": "jsts", ".cjs": "jsts", + ".ts": "jsts", ".tsx": "jsts", ".mts": "jsts", ".cts": "jsts", + ".vue": "jsts", ".svelte": "jsts", ".astro": "jsts", + # JVM interop + ".java": "jvm", ".kt": "jvm", ".kts": "jvm", + ".scala": "jvm", ".groovy": "jvm", ".gradle": "jvm", + # C-family: shared headers, Objective-C/C++ mix, Swift↔ObjC bridging + ".c": "native", ".h": "native", ".cpp": "native", ".cc": "native", + ".cxx": "native", ".hpp": "native", ".cu": "native", ".cuh": "native", + ".metal": "native", ".m": "native", ".mm": "native", ".swift": "native", + # Single-language families + ".py": "python", + ".go": "go", + ".rs": "rust", + ".rb": "ruby", ".rake": "ruby", + ".php": "php", ".phtml": "php", ".php3": "php", ".php4": "php", + ".php5": "php", ".php7": "php", ".phps": "php", + ".cs": "dotnet", ".razor": "dotnet", ".cshtml": "dotnet", ".xaml": "dotnet", + ".lua": "lua", ".luau": "lua", + ".zig": "zig", + ".ex": "elixir", ".exs": "elixir", + ".jl": "julia", + ".dart": "dart", + ".sh": "shell", ".bash": "shell", + ".ps1": "powershell", ".psm1": "powershell", ".psd1": "powershell", +} -def _sv_child(node, type_name: str) -> object | None: - if node is None: +def _lang_family(source_file: object) -> str | None: + """Interop family of the file's language, or None when unknown/not code.""" + if not source_file: return None - for child in node.children: - if child.type == type_name: - return child - return None + return _LANG_FAMILY_BY_EXT.get(Path(str(source_file)).suffix.lower()) -_SV_BUILTIN_TYPES = frozenset({ - "bit", "logic", "reg", "wire", "int", "integer", "shortint", "longint", - "byte", "time", "real", "shortreal", "void", "string", "type", "event", - "mailbox", "semaphore", "process", "chandle", -}) - -_SV_NON_TYPE_WORDS = frozenset({ - "return", "if", "else", "for", "foreach", "while", "case", "begin", "end", - "function", "task", "class", "endclass", "endfunction", "endtask", -}) - -# One level of balanced parens (e.g. `Foo #(Bar #(int))`) — bounded so malformed -# input cannot trigger pathological backtracking. -_SV_PARENS_INNER = r"(?:[^()]|\([^()]*\))*" -_SV_PARENS = r"\(" + _SV_PARENS_INNER + r"\)" +def _node_label_key(node: dict, fold: bool = False) -> str: + label = str(node.get("label", "")).strip() + key = re.sub(r"[^a-zA-Z0-9]+", "", label) + return key.lower() if fold else key -_SV_FUNC_RE = re.compile( - r"\bfunction\s+([A-Za-z_]\w*(?:\s*#\s*" + _SV_PARENS + r")?)\s+(\w+)\s*" - r"\((" + _SV_PARENS_INNER + r")\)\s*;", - re.MULTILINE, -) -_SV_PARAM_RE = re.compile( - r"\s*(?:input|output|inout|ref|const\s+ref)?\s*" - r"([A-Za-z_]\w*(?:\s*#\s*" + _SV_PARENS + r")?)\s+\w+" -) +def _is_top_level_function_definition(node: dict) -> bool: + """A free/top-level function def (label ``name()``), not a method or type. + Methods carry a leading dot (``.foo()``) or a qualifier (``Class.foo()``); + excluding those keeps a bare-name reference from binding to a receiver-scoped + method, which the receiver-typed resolvers own (#1781). + """ + label = str(node.get("label", "")).strip() + return ( + node.get("file_type") == "code" + and label.endswith(")") + and not label.startswith(".") + and "." not in label + ) -def _sv_strip_comments(text: str) -> str: - text = re.sub(r"/\*.*?\*/", "", text, flags=re.DOTALL) - return re.sub(r"//.*", "", text) +def _rewire_unique_stub_nodes(nodes: list[dict], edges: list[dict]) -> None: + """Map unresolved no-source stubs to a unique real definition with the same label.""" + real_by_label: dict[str, list[dict]] = {} # exact-case type-like (all languages) + real_by_label_ci: dict[str, list[dict]] = {} # case-INSENSITIVE-language reals only + func_by_label: dict[str, list[dict]] = {} # top-level function defs (#1781) + stubs: list[dict] = [] -def _sv_split_type_list(text: str) -> list[str]: - parts: list[str] = [] - depth = 0 - start = 0 - for idx, ch in enumerate(text): - if ch == "(": - depth += 1 - elif ch == ")": - depth = max(0, depth - 1) - elif ch == "," and depth == 0: - item = text[start:idx].strip() - if item: - parts.append(item) - start = idx + 1 - item = text[start:].strip() - if item: - parts.append(item) - return parts + for node in nodes: + key = _node_label_key(node) + if not key: + continue + if node.get("source_file"): + if _is_type_like_definition(node): + # Match stubs case-SENSITIVELY: a `Path` reference must not rewire to a + # `PATH` env var (#1581). Fold only for genuinely case-insensitive + # languages, where `foo` legitimately resolves to `Foo`. + real_by_label.setdefault(key, []).append(node) + if _lang_is_case_insensitive(node.get("source_file")): + real_by_label_ci.setdefault( + _node_label_key(node, fold=True), []).append(node) + elif _is_top_level_function_definition(node): + func_by_label.setdefault(key, []).append(node) + continue + stubs.append(node) + # Language families referencing each stub, for the function-merge guard (#1781): + # a cross-module `references` edge to a function used to dangle on a sourceless + # name-only stub because functions were excluded as rewire targets. We now allow + # a UNIQUE function definition to absorb it, but only when it shares a language + # family with the stub's referrers — so a Python `get_db` reference can't bind to + # a unique Go `get_db()` (mirrors the #1718/#1749 interop guard). + stub_ids = {str(s.get("id")) for s in stubs if s.get("id")} + stub_families: dict[str, set] = {} + supertype_stub_ids: set[str] = set() # stubs used as a base type — never a function + _SUPERTYPE_RELATIONS = {"inherits", "implements", "extends"} + for edge in edges: + rel = edge.get("relation") + for endpoint in ("source", "target"): + nid = edge.get(endpoint) + if nid in stub_ids: + fam = _lang_family(edge.get("source_file")) + if fam is not None: + stub_families.setdefault(str(nid), set()).add(fam) + # A stub referenced as a supertype must resolve to a class/type, + # not a same-named function (you don't inherit from a function). + if endpoint == "target" and rel in _SUPERTYPE_RELATIONS: + supertype_stub_ids.add(str(nid)) -def _sv_collect_type_refs(type_text: str, generic: bool = False, - skip: frozenset[str] = frozenset()) -> list[tuple[str, str]]: - refs: list[tuple[str, str]] = [] - text = type_text.strip() - if not text: - return refs - head = re.match(r"([A-Za-z_]\w*)", text) - if head: - name = head.group(1) - # `skip` carries the enclosing class's `#(type T = ...)` parameters so - # they are not mistaken for referenced types. - if name not in _SV_BUILTIN_TYPES and name not in _SV_NON_TYPE_WORDS and name not in skip: - refs.append((name, "generic_arg" if generic else "type")) - params = re.search(r"#\s*\((" + _SV_PARENS_INNER + r")\)", text) - if params: - for arg in _sv_split_type_list(params.group(1)): - refs.extend(_sv_collect_type_refs(arg, generic=True, skip=skip)) - return refs - - -def _augment_systemverilog_semantics( - raw: str, - stem: str, - str_path: str, - file_nid: str, - nodes: list[dict], - edges: list[dict], - seen_ids: set[str], -) -> None: - label_to_nid = {node["label"]: node["id"] for node in nodes} - - def line_for(offset: int) -> int: - return raw.count("\n", 0, offset) + 1 + remap: dict[str, str] = {} + for stub in stubs: + stub_id = str(stub.get("id", "")) + if not stub_id: + continue + candidates = real_by_label.get(_node_label_key(stub), []) + if len(candidates) != 1: + # No unique exact type match — fall back to a case-insensitive match, but + # only against case-insensitive-language definitions (so a case-sensitive + # `PATH` can never absorb a `Path` reference). + candidates = real_by_label_ci.get(_node_label_key(stub, fold=True), []) + if len(candidates) != 1: + # #1781: no unique type — try a unique top-level FUNCTION definition, + # gated by (a) the stub not being used as a supertype and (b) a + # language-family match with the stub's referrers. + fcands = func_by_label.get(_node_label_key(stub), []) + if len(fcands) == 1 and stub_id not in supertype_stub_ids: + fams = stub_families.get(stub_id, set()) + cand_fam = _lang_family(fcands[0].get("source_file")) + if not fams or cand_fam is None or cand_fam in fams: + candidates = fcands + if len(candidates) != 1: + continue + target_id = candidates[0].get("id") + if isinstance(target_id, str) and target_id and target_id != stub_id: + remap[stub_id] = target_id - def add_node(nid: str, label: str, line: int) -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({"id": nid, "label": label, "file_type": "code", - "source_file": str_path, "source_location": f"L{line}", - "confidence_score": 1.0}) - label_to_nid[label] = nid - - def ensure_type(label: str, line: int) -> str: - if label in label_to_nid: - return label_to_nid[label] - nid = _make_id(stem, label) - add_node(nid, label, line) - return nid - - def add_edge(src: str, target_label: str, relation: str, line: int, context: str | None = None) -> None: - tgt = ensure_type(target_label, line) - edge = {"source": src, "target": tgt, "relation": relation, - "confidence": "EXTRACTED", "confidence_score": 1.0, - "source_file": str_path, "source_location": f"L{line}", "weight": 1.0} - if context: - edge["context"] = context - edges.append(edge) + if not remap: + return - text = _sv_strip_comments(raw) - # Consuming `endclass` (rather than a lookahead) makes each match own its - # terminator, so back-to-back or malformed classes cannot bleed bodies. - class_re = re.compile( - r"\b(?:(interface)\s+)?class\s+(\w+)([^;{]*)\s*;(.*?)\bendclass\b", - re.DOTALL, - ) - for match in class_re.finditer(text): - class_name = match.group(2) - header = match.group(3) or "" - body = match.group(4) or "" - line = line_for(match.start()) - # `#(type T = Payload)` declares `T` as a class type parameter, not a - # referenced type — collect these to skip below. - type_params = frozenset(re.findall(r"\btype\s+(\w+)", header)) - class_nid = _make_id(stem, class_name) - add_node(class_nid, class_name, line) - edges.append({"source": file_nid, "target": class_nid, "relation": "defines", - "confidence": "EXTRACTED", "confidence_score": 1.0, - "source_file": str_path, "source_location": f"L{line}", "weight": 1.0}) - - ext = re.search(r"\bextends\s+(\w+)", header) - if ext: - add_edge(class_nid, ext.group(1), "inherits", line) - impl = re.search(r"\bimplements\s+([^;{]+)", header) - if impl: - for iface_name in _sv_split_type_list(impl.group(1)): - add_edge(class_nid, iface_name.split("#", 1)[0].strip(), "implements", line) - - body_without_functions = re.sub( - r"\bfunction\b.*?\bendfunction\b", - lambda m: "\n" * m.group(0).count("\n"), - body, - flags=re.DOTALL, + by_id = {node.get("id"): node for node in nodes if node.get("id")} + csharp_scoped_relations = {"inherits", "implements", "references", "imports"} + for edge in edges: + is_csharp_scoped_edge = ( + str(edge.get("source_file", "")).endswith(".cs") + and edge.get("relation") in csharp_scoped_relations ) - # Optional leading class-property qualifiers (rand/local/protected/etc.) - # must be consumed: otherwise a qualified field like `rand Config x;` - # (three tokens) fails the ` ;` shape and its type reference - # is silently dropped. - for field in re.finditer(r"^\s*(?:(?:rand|randc|local|protected|static|const|automatic|var)\s+)*([A-Za-z_]\w*(?:\s*#\s*\([^;]+?\))?)\s+\w+\s*;", body_without_functions, re.MULTILINE): - # Count to the start of the type token (group 1), not the match - # start: `^\s*` consumes the leading newline(s), so field.start() - # would resolve to the class's line instead of the field's. - field_line = line + body_without_functions.count("\n", 0, field.start(1)) - for ref_name, role in _sv_collect_type_refs(field.group(1), skip=type_params): - add_edge(class_nid, ref_name, "references", field_line, "generic_arg" if role == "generic_arg" else "field") - - for fm in _SV_FUNC_RE.finditer(body): - return_type, func_name, params = fm.group(1), fm.group(2), fm.group(3) - func_line = line + body.count("\n", 0, fm.start()) - func_nid = _make_id(class_nid, func_name) - add_node(func_nid, func_name, func_line) - edges.append({"source": class_nid, "target": func_nid, "relation": "method", - "confidence": "EXTRACTED", "confidence_score": 1.0, - "source_file": str_path, "source_location": f"L{func_line}", "weight": 1.0}) - for ref_name, role in _sv_collect_type_refs(return_type, skip=type_params): - add_edge(func_nid, ref_name, "references", func_line, "generic_arg" if role == "generic_arg" else "return_type") - for param in _sv_split_type_list(params): - pm = _SV_PARAM_RE.match(param) - if not pm: - continue - for ref_name, role in _sv_collect_type_refs(pm.group(1), skip=type_params): - add_edge(func_nid, ref_name, "references", func_line, "generic_arg" if role == "generic_arg" else "parameter_type") - - -def extract_verilog(path: Path) -> dict: - """Extract modules, functions, tasks, package imports, instantiations, and - SystemVerilog class semantics (inherits/implements edges, field/parameter/ - return-type references) from .v/.sv files.""" - try: - import tree_sitter_verilog as tsverilog - from tree_sitter import Language, Parser - except ImportError: - return {"nodes": [], "edges": [], "error": "tree_sitter_verilog not installed"} - - try: - language = Language(tsverilog.language()) - parser = Parser(language) - source = path.read_bytes() - tree = parser.parse(source) - root = tree.root_node - except Exception as e: - return {"nodes": [], "edges": [], "error": str(e)} - - stem = _file_stem(path) - str_path = str(path) - nodes: list[dict] = [] - edges: list[dict] = [] - seen_ids: set[str] = set() - - def add_node(nid: str, label: str, line: int) -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({"id": nid, "label": label, "file_type": "code", - "source_file": str_path, "source_location": f"L{line}", - "confidence_score": 1.0}) - - def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", score: float = 1.0) -> None: - edges.append({"source": src, "target": tgt, "relation": relation, - "confidence": confidence, "confidence_score": score, - "source_file": str_path, "source_location": f"L{line}", "weight": 1.0}) - - file_nid = _make_id(str(path)) - add_node(file_nid, path.name, 1) - - def walk(node, module_nid: str | None = None) -> None: - t = node.type + source = edge.get("source") + if source in remap: + remapped_source = remap[str(source)] + if not ( + is_csharp_scoped_edge + and str(by_id.get(remapped_source, {}).get("source_file", "")).endswith(".cs") + ): + edge["source"] = remapped_source + target = edge.get("target") + if target in remap: + remapped_target = remap[str(target)] + if not ( + is_csharp_scoped_edge + and str(by_id.get(remapped_target, {}).get("source_file", "")).endswith(".cs") + ): + edge["target"] = remapped_target - # SystemVerilog class bodies are handled by _augment_systemverilog_semantics - # (regex over source text). Skip their subtrees so in-class methods are not - # double-emitted here — and with the wrong, return-type-derived name. - if t in ("class_declaration", "interface_class_declaration"): - return + referenced = {x for e in edges for x in (e.get("source"), e.get("target"))} + drop_ids = {stub_id for stub_id in remap if stub_id not in referenced} + nodes[:] = [node for node in nodes if node.get("id") not in drop_ids] - if t == "module_declaration": - mod_name = _sv_first_identifier(_sv_child(node, "module_header"), source) - if mod_name: - line = node.start_point[0] + 1 - nid = _make_id(stem, mod_name) - add_node(nid, mod_name, line) - add_edge(file_nid, nid, "defines", line) - for child in node.children: - walk(child, nid) - return - # `function_prototype` only appears inside class/interface-class bodies - # (skipped above) and nests its name differently; it is intentionally not - # handled here. - elif t == "function_declaration": - fn_body = _sv_child(node, "function_body_declaration") - func_name = _sv_first_identifier(_sv_child(fn_body, "function_identifier"), source) - if func_name: - line = node.start_point[0] + 1 - parent = module_nid or file_nid - nid = _make_id(parent, func_name) - add_node(nid, f"{func_name}()", line) - add_edge(parent, nid, "contains", line) - - elif t == "task_declaration": - tk_body = _sv_child(node, "task_body_declaration") - task_name = _sv_first_identifier(_sv_child(tk_body, "task_identifier"), source) - if task_name: - line = node.start_point[0] + 1 - parent = module_nid or file_nid - nid = _make_id(parent, task_name) - add_node(nid, task_name, line) - add_edge(parent, nid, "contains", line) - - elif t == "package_import_declaration": - for child in node.children: - if child.type == "package_import_item": - pkg_text = _read_text(child, source) - pkg_name = pkg_text.split("::")[0].strip() - if pkg_name: - line = node.start_point[0] + 1 - tgt_nid = _make_id(pkg_name) - add_node(tgt_nid, pkg_name, line) - src_nid = module_nid or file_nid - add_edge(src_nid, tgt_nid, "imports_from", line) - - elif t in ("module_instantiation", "checker_instantiation"): - # `leaf u_leaf();` parses as checker_instantiation in 1.0.3; - # module_instantiation (when it occurs) exposes a `module_type` field. - # Both reduce to the first identifier under the node — the instantiated - # type, not the instance name (which appears later). - if module_nid: - type_node = node.child_by_field_name("module_type") - inst_type = (_read_text(type_node, source).strip() if type_node - else _sv_first_identifier(node, source)) - if inst_type: - line = node.start_point[0] + 1 - tgt_nid = _make_id(inst_type) - add_node(tgt_nid, inst_type, line) - add_edge(module_nid, tgt_nid, "instantiates", line) +def _augment_js_reexport_edges( + paths: list[Path], + nodes: list[dict], + edges: list[dict], + root: Path, +) -> None: + """Compatibility wrapper for the JS/TS symbol-resolution post-pass.""" + facts = _SymbolResolutionFacts() + _collect_js_symbol_resolution_facts(paths, facts) + _apply_symbol_resolution_facts(paths, nodes, edges, root, facts) - for child in node.children: - walk(child, module_nid) - - walk(root) - _augment_systemverilog_semantics( - source.decode("utf-8", errors="replace"), - stem, - str_path, - file_nid, - nodes, - edges, - seen_ids, - ) - return {"nodes": nodes, "edges": edges} +# Header / implementation file-extension pairing for the decl/def class merge. -def extract_sql(path: Path, content: str | bytes | None = None) -> dict: - """Extract tables, views, functions, and relationships from .sql files via tree-sitter.""" - try: - import tree_sitter_sql as tssql - from tree_sitter import Language, Parser - except ImportError: - return {"nodes": [], "edges": [], "error": "tree_sitter_sql not installed. Run: pip install tree-sitter-sql"} - try: - language = Language(tssql.language()) - parser = Parser(language) - source = ( - content.encode("utf-8") if isinstance(content, str) - else content if content is not None - else path.read_bytes() - ) - tree = parser.parse(source) - root = tree.root_node - except Exception as e: - return {"nodes": [], "edges": [], "error": str(e)} +def _merge_swift_extensions( + per_file: list[dict], + all_nodes: list[dict], + all_edges: list[dict], +) -> None: + """Collapse cross-file Swift `extension Foo` nodes into the canonical `Foo`. + tree-sitter-swift reuses `class_declaration` for both `class Foo` and + `extension Foo`, and node ids carry the file stem, so each file that + extends `Foo` produces its own `Foo` node. The match is done by label: + when exactly one non-extension declaration shares the label, extension + nodes redirect onto it. Extensions of types outside the corpus (no match) + and ambiguous labels (more than one match) are left untouched — picking + arbitrarily would invent edges. + """ + extension_nids: set[str] = set() + extension_labels: dict[str, str] = {} + for result in per_file: + for ext in result.get("swift_extensions", []) or []: + extension_nids.add(ext["nid"]) + extension_labels[ext["nid"]] = ext["label"] - stem = _file_stem(path) - str_path = str(path) - file_nid = _make_id(str_path) - nodes: list[dict] = [{"id": file_nid, "label": path.name, "file_type": "code", - "source_file": str_path, "source_location": None}] - edges: list[dict] = [] - seen_ids: set[str] = {file_nid} - table_nids: dict[str, str] = {} # name → nid for reference resolution + if not extension_nids: + return - def _read(n) -> str: - return source[n.start_byte:n.end_byte].decode("utf-8", errors="replace") + label_to_canonical: dict[str, list[str]] = {} + for n in all_nodes: + if n.get("id") in extension_nids: + continue + label = n.get("label") + if not label: + continue + label_to_canonical.setdefault(label, []).append(n["id"]) - def _obj_name(n) -> str | None: - for c in n.children: - if c.type == "object_reference": - return _read(c) - return None + remap: dict[str, str] = {} + for ext_nid in extension_nids: + candidates = label_to_canonical.get(extension_labels[ext_nid], []) + if len(candidates) != 1: + continue + canonical_nid = candidates[0] + if canonical_nid != ext_nid: + remap[ext_nid] = canonical_nid - def _add_node(nid: str, label: str, line: int) -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({"id": nid, "label": label, "file_type": "code", - "source_file": str_path, "source_location": f"L{line}"}) - edges.append({"source": file_nid, "target": nid, "relation": "contains", - "confidence": "EXTRACTED", "source_file": str_path, - "source_location": f"L{line}", "weight": 1.0}) - - def _add_edge(src: str, tgt: str, relation: str, line: int) -> None: - edges.append({"source": src, "target": tgt, "relation": relation, - "confidence": "EXTRACTED", "source_file": str_path, - "source_location": f"L{line}", "weight": 1.0}) - - def walk(node) -> None: - t = node.type - line = node.start_point[0] + 1 + if not remap: + return - if t == "create_table": - name = _obj_name(node) - if name: - nid = _make_id(stem, name) - _add_node(nid, name, line) - table_nids[name.lower()] = nid - # Foreign key REFERENCES - for col in node.children: - if col.type == "column_definitions": - has_error = any(cd.type == "ERROR" for cd in col.children) - seen_refs: set[str] = set() - for cd in col.children: - if cd.type == "column_definition": - # Inline column-level REFERENCES - ref_name: str | None = None - found_ref = False - for cc in cd.children: - if cc.type == "keyword_references": - found_ref = True - elif found_ref and cc.type == "object_reference": - ref_name = _read(cc) - break - if ref_name: - ref_nid = table_nids.get(ref_name.lower()) or _make_id(stem, ref_name) - _add_edge(nid, ref_nid, "references", line) - seen_refs.add(ref_name.lower()) - elif cd.type == "constraints": - # Table-level FOREIGN KEY ... REFERENCES ... constraints - for constraint in cd.children: - if constraint.type != "constraint": - continue - ref_name = None - found_ref = False - for cc in constraint.children: - if cc.type == "keyword_references": - found_ref = True - elif found_ref and cc.type == "object_reference": - ref_name = _read(cc) - break - if ref_name: - ref_nid = table_nids.get(ref_name.lower()) or _make_id(stem, ref_name) - _add_edge(nid, ref_nid, "references", line) - seen_refs.add(ref_name.lower()) - if has_error: - # Dialect-specific syntax (e.g. Firebird COMPUTED BY) causes ERROR - # nodes that make the parser drop the trailing constraints block. - # Regex-scan the raw column_definitions text as fallback. - col_text = _read(col) - for rm in re.finditer(r"\bREFERENCES\s+([\w$]+)", col_text, re.IGNORECASE): - ref_name = rm.group(1) - if ref_name.lower() not in seen_refs: - ref_nid = table_nids.get(ref_name.lower()) or _make_id(stem, ref_name) - _add_edge(nid, ref_nid, "references", line) - seen_refs.add(ref_name.lower()) - - elif t == "create_view": - name = _obj_name(node) - if name: - nid = _make_id(stem, name) - _add_node(nid, name, line) - table_nids[name.lower()] = nid - # FROM/JOIN table references inside view body - _walk_from_refs(node, nid, line) - - elif t == "create_function": - name = _obj_name(node) - if name: - nid = _make_id(stem, name) - _add_node(nid, f"{name}()", line) - _walk_from_refs(node, nid, line) - - elif t == "create_procedure": - name = _obj_name(node) - if name: - nid = _make_id(stem, name) - _add_node(nid, f"{name}()", line) - _walk_from_refs(node, nid, line) - - elif t == "alter_table": - name = _obj_name(node) - if name: - src_nid = table_nids.get(name.lower()) - if not src_nid: - src_nid = _make_id(stem, name) - _add_node(src_nid, name, line) - table_nids[name.lower()] = src_nid - for child in node.children: - if child.type == "add_constraint": - for cc in child.children: - if cc.type != "constraint": - continue - found_ref = False - ref_name: str | None = None - for ccc in cc.children: - if ccc.type == "keyword_references": - found_ref = True - elif found_ref and ccc.type == "object_reference": - ref_name = _read(ccc) - break - if ref_name: - ref_nid = table_nids.get(ref_name.lower()) - if not ref_nid: - ref_nid = _make_id(stem, ref_name) - _add_edge(src_nid, ref_nid, "references", line) - - elif t == "create_trigger": - trig_name: str | None = None - tbl_name: str | None = None - after_trigger = False - after_for = False - for c in node.children: - if c.type == "keyword_trigger": - after_trigger = True - elif after_trigger and not trig_name and c.type == "object_reference": - trig_name = _read(c) - elif c.type == "keyword_for": - after_for = True - elif after_for and not tbl_name and c.type == "object_reference": - tbl_name = _read(c) - if trig_name: - trig_nid = _make_id(stem, trig_name) - _add_node(trig_nid, trig_name, line) - if tbl_name: - tbl_nid = table_nids.get(tbl_name.lower()) or _make_id(stem, tbl_name) - _add_edge(trig_nid, tbl_nid, "triggers", line) - - elif t == "fb_proc_or_trigger": - text = _read(node) - m = re.match( - r"CREATE\s+(?:OR\s+(?:REPLACE|ALTER)\s+)?" - r"(PROCEDURE|TRIGGER|FUNCTION)\s+([\w$]+)", - text, re.IGNORECASE, - ) - if m: - obj_type = m.group(1).upper() - obj_name = m.group(2) - obj_nid = _make_id(stem, obj_name) - label = obj_name if obj_type == "TRIGGER" else f"{obj_name}()" - _add_node(obj_nid, label, line) - if obj_type == "TRIGGER": - fm = re.search(r"\bFOR\s+([\w$]+)", text, re.IGNORECASE) - if fm: - tbl = fm.group(1) - tbl_nid = table_nids.get(tbl.lower()) or _make_id(stem, tbl) - _add_edge(obj_nid, tbl_nid, "triggers", line) - _NON_TABLES = { - "select", "where", "set", "dual", "null", "true", "false", - "first", "skip", "rows", "next", "only", "lateral", - } - seen_tbls: set[str] = set() - for rm in re.finditer(r"\b(?:FROM|JOIN|INTO)\s+([\w$]+)", text, re.IGNORECASE): - tbl = rm.group(1) - if tbl.lower() not in _NON_TABLES and tbl.lower() not in seen_tbls: - seen_tbls.add(tbl.lower()) - tbl_nid = table_nids.get(tbl.lower()) or _make_id(stem, tbl) - _add_edge(obj_nid, tbl_nid, "reads_from", line) - for rm in re.finditer(r"\bUPDATE\s+([\w$]+)", text, re.IGNORECASE): - tbl = rm.group(1) - if tbl.lower() not in _NON_TABLES and tbl.lower() not in seen_tbls: - seen_tbls.add(tbl.lower()) - tbl_nid = table_nids.get(tbl.lower()) or _make_id(stem, tbl) - _add_edge(obj_nid, tbl_nid, "reads_from", line) + all_nodes[:] = [n for n in all_nodes if n.get("id") not in remap] - for child in node.children: - walk(child) - - def _walk_from_refs(node, caller_nid: str, line: int) -> None: - """Recursively find FROM/JOIN table references inside a node.""" - if node.type in ("from", "join"): - for c in node.children: - if c.type == "relation": - for cc in c.children: - if cc.type == "object_reference": - tbl = _read(cc) - tbl_nid = _make_id(stem, tbl) - _add_edge(caller_nid, tbl_nid, "reads_from", - c.start_point[0] + 1) - for child in node.children: - _walk_from_refs(child, caller_nid, line) - - for stmt in root.children: - if stmt.type == "statement": - for child in stmt.children: - walk(child) - elif stmt.type in ("fb_proc_or_trigger", "set_term", "declare_external_function"): - walk(stmt) - - # Global regex fallback: catch any REFERENCES missed due to ERROR nodes in the parse tree - # (e.g. Firebird COMPUTED BY columns push constraints out of the tree entirely). - # Snapshot after tree walk so we don't re-emit edges already captured above. - emitted = {(e["source"], e["target"]) for e in edges if e["relation"] == "references"} - src_text = source.decode("utf-8", errors="replace") - for m in re.finditer(r"CREATE\s+TABLE\s+([\w$]+)\s*\(", src_text, re.IGNORECASE): - tbl_name = m.group(1) - tbl_nid = table_nids.get(tbl_name.lower()) - if tbl_nid is None: + # Each extension file's `contains` edge ends up pointing at the canonical + # type — multiple files containing the same node is the intended shape: + # the type owns the methods, the files own their slice. Self-loops are + # dropped (e.g. an in-file extension method whose call already pointed at + # the canonical type). + rewritten: list[dict] = [] + seen_keys: set[tuple] = set() + for e in all_edges: + src = remap.get(e.get("source"), e.get("source")) + tgt = remap.get(e.get("target"), e.get("target")) + if src == tgt: + continue + e["source"] = src + e["target"] = tgt + key = (src, tgt, e.get("relation"), e.get("source_file"), e.get("source_location")) + if key in seen_keys: continue - tbl_line = src_text[: m.start()].count("\n") + 1 - tail = src_text[m.start():] - end = re.search(r"(?:^|\n)(?:CREATE|SET\s+TERM|ALTER)\s", tail[1:], re.IGNORECASE) - block = tail[: end.start() + 1] if end else tail - for rm in re.finditer(r"\bREFERENCES\s+([\w$]+)", block, re.IGNORECASE): - ref_name = rm.group(1) - ref_nid = table_nids.get(ref_name.lower()) or _make_id(stem, ref_name) - if (tbl_nid, ref_nid) not in emitted: - _add_edge(tbl_nid, ref_nid, "references", tbl_line) - emitted.add((tbl_nid, ref_nid)) + seen_keys.add(key) + rewritten.append(e) + all_edges[:] = rewritten - return {"nodes": nodes, "edges": edges} +def _resolve_swift_member_calls( + per_file: list[dict], + all_nodes: list[dict], + all_edges: list[dict], +) -> None: + """Resolve cross-file Swift member calls (``recv.method()``) to the real + definition of the receiver's type (#1356). -def extract_lua(path: Path) -> dict: - """Extract functions, methods, require() imports, and calls from a .lua file.""" - return _extract_generic(path, _LUA_CONFIG) + The shared cross-file call pass drops every ``is_member_call`` because a bare + method name (``update``) collides across the corpus and inflates god-nodes + (#543/#1219). Swift extractors record the receiver of each member call and a + per-file ``name -> type`` table (``swift_type_table``); this pass uses them to + type the receiver, then emits an edge ONLY when that type name resolves to + exactly one definition. A type-qualified call (``Type.staticMethod()``) is + EXTRACTED (the type is named explicitly in source); an instance call typed via + local inference (``obj.method()``) is INFERRED. The shared-pass member-call drop + stays intact: this is purely additive and fires only on receiver-typed Swift calls. + Must run after id-disambiguation so node ids and caller_nids are final. + """ + type_table_by_file: dict[str, dict[str, str]] = {} + for result in per_file: + tt = result.get("swift_type_table") + if tt and tt.get("path"): + type_table_by_file[tt["path"]] = tt.get("table", {}) + if not type_table_by_file: + return -def extract_swift(path: Path) -> dict: - """Extract classes, structs, protocols, functions, imports, and calls from a .swift file.""" - return _extract_generic(path, _SWIFT_CONFIG) + def _key(label: str) -> str: + return re.sub(r"[^a-zA-Z0-9]+", "", str(label)).lower() + # A genuine Swift type is the target of a `contains` edge from its file node. + # Bare type references create a same-label shadow node (via ensure_named_node) + # that carries a source_file but is NOT contained; excluding non-contained + # nodes keeps that shadow from making a real type name look ambiguous. + contained = {e.get("target") for e in all_edges if e.get("relation") == "contains"} -# ── Julia extractor (custom walk) ──────────────────────────────────────────── + # Type name -> definition node ids (real, source-backed, type-like defs only). + # len != 1 is the god-node guard: an ambiguous type name bails. + type_def_nids: dict[str, list[str]] = {} + node_by_id: dict[str, dict] = {} + for n in all_nodes: + node_by_id[n.get("id")] = n + if n.get("source_file") and n.get("id") in contained and _is_type_like_definition(n): + type_def_nids.setdefault(_key(n.get("label", "")), []).append(n["id"]) -def extract_julia(path: Path) -> dict: - """Extract modules, structs, functions, imports, and calls from a .jl file.""" - try: - import tree_sitter_julia as tsjulia - from tree_sitter import Language, Parser - except ImportError: - return {"nodes": [], "edges": [], "error": "tree-sitter-julia not installed"} + # (type_node_id, method_key) -> method_node_id, from `method` edges. + method_index: dict[tuple[str, str], str] = {} + for e in all_edges: + if e.get("relation") != "method": + continue + src, tgt = e.get("source"), e.get("target") + tnode = node_by_id.get(tgt) + if tnode is not None: + method_index[(src, _key(tnode.get("label", "")))] = tgt - try: - language = Language(tsjulia.language()) - parser = Parser(language) - source = path.read_bytes() - tree = parser.parse(source) - root = tree.root_node - except Exception as e: - return {"nodes": [], "edges": [], "error": str(e)} + all_raw_calls: list[dict] = [] + for result in per_file: + all_raw_calls.extend(result.get("raw_calls", [])) - stem = _file_stem(path) - str_path = str(path) - nodes: list[dict] = [] - edges: list[dict] = [] - seen_ids: set[str] = set() - function_bodies: list[tuple[str, object]] = [] - - def add_node(nid: str, label: str, line: int) -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": label, - "file_type": "code", - "source_file": str_path, - "source_location": f"L{line}", - }) - - def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", weight: float = 1.0, - context: str | None = None) -> None: - edge = { - "source": src, - "target": tgt, + existing_pairs = {(e.get("source"), e.get("target")) for e in all_edges} + for rc in all_raw_calls: + if not rc.get("is_member_call"): + continue + receiver = rc.get("receiver") + callee = rc.get("callee") + if not receiver or not callee: + continue + # Determine the receiver's type. An upper-cased receiver is itself a type + # (Type.staticMethod(), Singleton.shared.x()); otherwise look it up in the + # declaring file's local type table. + if receiver[:1].isupper(): + type_name = receiver + type_qualified = True + else: + type_name = type_table_by_file.get(rc.get("source_file", ""), {}).get(receiver) + type_qualified = False + if not type_name: + continue + # A builtin receiver type (Data, NSLock, DispatchQueue, ...) must not + # resolve to a same-named user symbol — the cross-file CALL resolver and + # the TS/Python member-call resolvers already skip these globals (#1726); + # do the same for Swift (#2147). + if type_name in _LANGUAGE_BUILTIN_GLOBALS: + continue + type_defs = type_def_nids.get(_key(type_name), []) + if len(type_defs) != 1: # ambiguous or absent -> bail (god-node guard) + continue + type_nid = type_defs[0] + caller = rc.get("caller_nid") + if not caller: + continue + method_nid = method_index.get((type_nid, _key(callee))) + target = method_nid or type_nid + relation = "calls" if method_nid else "references" + if target == caller or (caller, target) in existing_pairs: + continue + existing_pairs.add((caller, target)) + # A type-qualified call (`Type.staticMethod()`) names the receiver type + # explicitly in source, so it is an exact reference — EXTRACTED, matching + # the Python qualified-class-method pass (#1533). An instance call whose + # receiver type came from local inference (`obj.method()`) stays INFERRED. + all_edges.append({ + "source": caller, + "target": target, "relation": relation, - "confidence": confidence, - "source_file": str_path, - "source_location": f"L{line}", - "weight": weight, - } - if context: - edge["context"] = context - edges.append(edge) - - file_nid = _make_id(str(path)) - add_node(file_nid, path.name, 1) - - def ensure_named_node(name: str, line: int) -> str: - nid = _make_id(stem, name) - if nid in seen_ids: - return nid - nid = _make_id(name) - if nid not in seen_ids: - # The name isn't defined in this file, so this is a cross-file reference - # (e.g. a `Thing` type annotation imported from another module). Emit a - # SOURCELESS stub — like the inheritance-base path below — so the - # corpus-level rewire can collapse it onto the real definition. A sourced - # stub here makes _disambiguate_colliding_node_ids bake the referencing - # file's path (with extension) into the id and blocks the rewire, which is - # the phantom-duplicate-node bug (#1402). - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": name, - "file_type": "code", - "source_file": "", - "source_location": "", - "origin_file": str_path, - }) - return nid - - def _func_name_from_signature(sig_node) -> str | None: - """Extract function name from a Julia signature node (call_expression > identifier).""" - for child in sig_node.children: - if child.type == "call_expression": - callee = child.children[0] if child.children else None - if callee and callee.type == "identifier": - return _read_text(callee, source) - return None + "context": "call", + "confidence": "EXTRACTED" if type_qualified else "INFERRED", + "confidence_score": 1.0 if type_qualified else 0.8, + "source_file": rc.get("source_file", ""), + "source_location": rc.get("source_location"), + "weight": 1.0, + }) - def walk_calls(body_node, func_nid: str) -> None: - if body_node is None: - return - t = body_node.type - if t in ("function_definition", "short_function_definition"): - return - if t == "call_expression" and body_node.children: - callee = body_node.children[0] - # Direct call: foo(...) - if callee.type == "identifier": - callee_name = _read_text(callee, source) - target_nid = _make_id(stem, callee_name) - add_edge(func_nid, target_nid, "calls", body_node.start_point[0] + 1, - confidence="EXTRACTED", context="call") - # Method call: obj.method(...) - elif callee.type == "field_expression" and len(callee.children) >= 3: - method_node = callee.children[-1] - method_name = _read_text(method_node, source) - target_nid = _make_id(stem, method_name) - add_edge(func_nid, target_nid, "calls", body_node.start_point[0] + 1, - confidence="EXTRACTED", context="call") - for child in body_node.children: - walk_calls(child, func_nid) - def walk(node, scope_nid: str) -> None: - t = node.type +def _resolve_python_member_calls( + per_file: list[dict], + all_nodes: list[dict], + all_edges: list[dict], +) -> None: + """Resolve cross-file Python qualified class-method calls (``ClassName.method()``) + to the class-qualified method node (#1446). - # Module - if t == "module_definition": - name_node = next((c for c in node.children if c.type == "identifier"), None) - if name_node: - mod_name = _read_text(name_node, source) - mod_nid = _make_id(stem, mod_name) - line = node.start_point[0] + 1 - add_node(mod_nid, mod_name, line) - add_edge(file_nid, mod_nid, "defines", line) - for child in node.children: - walk(child, mod_nid) - return + The shared cross-file call pass drops every ``is_member_call`` because a bare + method name (``log``) collides across the corpus and inflates god-nodes + (#543/#1219). That guard is right for *instance* calls (``obj.method()``) but + misses *class-qualified* calls (``ClassName.method()``), where the receiver is + an explicitly-named class — an exact, unambiguous reference. This pass uses the + receiver captured by the extractor, and when it is a capitalized name resolving + to exactly one class node that owns the called method, emits an EXTRACTED + ``calls`` edge. Purely additive (only member calls the shared pass skipped), + with a single-definition god-node guard. - # Struct (struct / mutable struct — both map to struct_definition in tree-sitter-julia) - if t == "struct_definition": - # type_head may contain: identifier (simple) or binary_expression (Foo <: Bar) - type_head = next((c for c in node.children if c.type == "type_head"), None) - if not type_head: - return - struct_name: str | None = None - super_name: str | None = None - bin_expr = next((c for c in type_head.children if c.type == "binary_expression"), None) - if bin_expr: - identifiers = [c for c in bin_expr.children if c.type == "identifier"] - if identifiers: - struct_name = _read_text(identifiers[0], source) - if len(identifiers) >= 2: - super_name = _read_text(identifiers[-1], source) - else: - name_node = next((c for c in type_head.children if c.type == "identifier"), None) - if name_node: - struct_name = _read_text(name_node, source) - if not struct_name: - return - struct_nid = _make_id(stem, struct_name) - line = node.start_point[0] + 1 - add_node(struct_nid, struct_name, line) - add_edge(scope_nid, struct_nid, "defines", line) - if super_name: - add_edge(struct_nid, ensure_named_node(super_name, line), - "inherits", line, confidence="EXTRACTED") - # Field types: each `name::Type` lowers to a typed_expression child of struct_definition - for child in node.children: - if child.type == "typed_expression": - type_ids = [c for c in child.children if c.type == "identifier"] - if len(type_ids) >= 2: - field_line = child.start_point[0] + 1 - type_name = _read_text(type_ids[-1], source) - type_nid = ensure_named_node(type_name, field_line) - edges.append(_semantic_reference_edge( - struct_nid, type_nid, "field", str_path, field_line)) - return + Must run after id-disambiguation so node ids and caller_nids are final. + """ + def _key(label: str) -> str: + return re.sub(r"[^a-zA-Z0-9]+", "", str(label)).lower() - # Abstract type - if t == "abstract_definition": - # type_head > identifier - type_head = next((c for c in node.children if c.type == "type_head"), None) - if type_head: - name_node = next((c for c in type_head.children if c.type == "identifier"), None) - if name_node: - abs_name = _read_text(name_node, source) - abs_nid = _make_id(stem, abs_name) - line = node.start_point[0] + 1 - add_node(abs_nid, abs_name, line) - add_edge(scope_nid, abs_nid, "defines", line) - return + node_by_id: dict[str, dict] = {n.get("id"): n for n in all_nodes} - # Function: function foo(...) ... end - if t == "function_definition": - sig_node = next((c for c in node.children if c.type == "signature"), None) - if sig_node: - func_name = _func_name_from_signature(sig_node) - if func_name: - func_nid = _make_id(stem, func_name) - line = node.start_point[0] + 1 - add_node(func_nid, f"{func_name}()", line) - add_edge(scope_nid, func_nid, "defines", line) - function_bodies.append((func_nid, node)) - return + # A class owns methods: it is the source of one or more `method` edges. Index + # class label -> owning class node ids (len != 1 is the god-node guard), and + # (class_node_id, method_key) -> method_node_id. + class_def_nids: dict[str, list[str]] = {} + method_index: dict[tuple[str, str], str] = {} + for e in all_edges: + if e.get("relation") != "method": + continue + src, tgt = e.get("source"), e.get("target") + cnode = node_by_id.get(src) + if cnode is not None: + class_def_nids.setdefault(_key(cnode.get("label", "")), []).append(src) + tnode = node_by_id.get(tgt) + if tnode is not None: + method_index[(src, _key(tnode.get("label", "")))] = tgt + # A class with N methods produced N entries; collapse to a unique set. (No + # early return when there are no classes: the module arm below resolves + # `module.func()` where the callable is a plain function, not a method.) + for k in list(class_def_nids): + class_def_nids[k] = sorted(set(class_def_nids[k])) - # Short function: foo(x) = expr - if t == "assignment": - lhs = node.children[0] if node.children else None - if lhs and lhs.type == "call_expression" and lhs.children: - callee = lhs.children[0] - if callee.type == "identifier": - func_name = _read_text(callee, source) - func_nid = _make_id(stem, func_name) - line = node.start_point[0] + 1 - add_node(func_nid, f"{func_name}()", line) - add_edge(scope_nid, func_nid, "defines", line) - # Only walk the RHS (index 2 after lhs and operator) to avoid self-loops - rhs = node.children[-1] if len(node.children) >= 3 else None - if rhs: - function_bodies.append((func_nid, rhs)) - return + all_raw_calls: list[dict] = [] + for result in per_file: + all_raw_calls.extend(result.get("raw_calls", [])) - # Using / Import - if t in ("using_statement", "import_statement"): - line = node.start_point[0] + 1 - - def _julia_mod_name(n): - # identifier (`Foo`), scoped_identifier (`Base.Threads`), or - # import_path (relative `..Sibling`) -> the module name. Only bare - # identifiers were handled, so qualified/relative imports — and the - # scoped package of a `selected_import` — were silently dropped. - if n.type == "import_path": - ids = [c for c in n.children if c.type == "identifier"] - return _read_text(ids[-1], source) if ids else None - if n.type in ("identifier", "scoped_identifier"): - return _read_text(n, source) - return None + # Module-alias arm index (#1883): `module.func()` where `module` is imported. + # Key on stable node ids, not source_file strings (source_file is relativized + # by the CLI id-remap pass but raw_calls keep their original path, so a string + # join would miss under an explicit cache_root). The `imports` edge's source + # is the caller's own file node; `contains` maps a file node to its children. + contains_children: dict[str, dict[str, list[str]]] = {} + file_of_node: dict[str, str] = {} + for e in all_edges: + if e.get("relation") == "contains": + src, tgt = e.get("source"), e.get("target") + tnode = node_by_id.get(tgt) + if tnode is not None: + contains_children.setdefault(src, {}).setdefault( + _key(tnode.get("label", "")), []).append(tgt) + file_of_node[tgt] = src + imported_by_filenode: dict[str, set[str]] = {} + # Local alias bound by `as` on a specific import edge (#2082): `from pkg import + # mod as alias` / `import pkg.mod as alias` bind `alias`, not `mod`'s own stem, + # to the module in the importing file. Keyed by (importing file, target module) + # so two files aliasing the same module differently each match their own. + import_alias_by_filenode: dict[str, dict[str, str]] = {} + for e in all_edges: + if e.get("relation") in ("imports", "imports_from"): + imported_by_filenode.setdefault(e.get("source"), set()).add(e.get("target")) + alias = e.get("local_alias") + if alias: + import_alias_by_filenode.setdefault(e.get("source"), {})[e.get("target")] = _key(alias) + + def _module_stem_key(nid: str) -> str: + n = node_by_id.get(nid) + if not n: + return "" + sf = n.get("source_file") or "" + stem = Path(sf).stem if sf else "" + return _key(stem or n.get("label", "")) - def _emit_import(name): - if not name: - return - imp_nid = _make_id(name) - add_node(imp_nid, name, line) - add_edge(scope_nid, imp_nid, "imports", line, context="import") + existing_pairs = {(e.get("source"), e.get("target")) for e in all_edges} - for child in node.children: - if child.type in ("identifier", "scoped_identifier", "import_path"): - _emit_import(_julia_mod_name(child)) - elif child.type == "selected_import": - # `import Base.Threads: nthreads` — the package (first named - # child) may itself be a scoped_identifier/import_path. - pkg = next( - (c for c in child.children - if c.type in ("identifier", "scoped_identifier", "import_path")), - None, - ) - if pkg is not None: - _emit_import(_julia_mod_name(pkg)) + def _emit_call(caller: str, target_nid: "str | None", rc: dict) -> None: + if not target_nid or target_nid == caller or (caller, target_nid) in existing_pairs: return + existing_pairs.add((caller, target_nid)) + # EXTRACTED: a qualified call (`ClassName.method()` or `module.func()`) is + # an explicit, unambiguous static reference resolved to exactly one + # definition (each arm applies a single-definition god-node guard). + all_edges.append({ + "source": caller, + "target": target_nid, + "relation": "calls", + "context": "call", + "confidence": "EXTRACTED", + "confidence_score": 1.0, + "source_file": rc.get("source_file", ""), + "source_location": rc.get("source_location"), + "weight": 1.0, + }) - for child in node.children: - walk(child, scope_nid) - - walk(root, file_nid) - - for func_nid, body_node in function_bodies: - # For function_definition nodes, walk children directly to avoid - # the boundary check returning early on the top-level node itself. - # Skip the "signature" child — it contains the function's own call_expression - # which would create a self-loop. - if body_node.type == "function_definition": - for child in body_node.children: - if child.type != "signature": - walk_calls(child, func_nid) + for rc in all_raw_calls: + if not rc.get("is_member_call"): + continue + receiver = rc.get("receiver") + callee = rc.get("callee") + caller = rc.get("caller_nid") + if not receiver or not callee or not caller: + continue + if receiver[:1].isupper(): + # Class arm (#1446): a capitalized receiver is a class reference; an + # instance (`self`, `obj`) never collides with a same-spelled class. + class_nids = class_def_nids.get(_key(receiver), []) + if len(class_nids) != 1: # absent or ambiguous -> bail (god-node guard) + continue + _emit_call(caller, method_index.get((class_nids[0], _key(callee))), rc) else: - walk_calls(body_node, func_nid) - - return {"nodes": nodes, "edges": edges} - - -_FORTRAN_CPP_EXTS = {".F", ".F90", ".F95", ".F03", ".F08"} - + # Module arm (#1883): a lowercase receiver may be an imported module. + # Resolve it against the modules imported into the caller's own file + # (so `self`/`obj`/local instances, which are not imported modules, + # never match), then to the single callable that module contains. A + # receiver also matches the local alias bound on that import edge + # (#2082), so an aliased import resolves the same as the bare name. + rkey = _key(receiver) + caller_file = file_of_node.get(caller) + file_aliases = import_alias_by_filenode.get(caller_file, {}) + mods = [t for t in imported_by_filenode.get(caller_file, ()) + if t in contains_children + and (_module_stem_key(t) == rkey or file_aliases.get(t) == rkey)] + if len(mods) != 1: # not an imported module, or ambiguous -> bail + continue + children = contains_children[mods[0]].get(_key(callee), []) + if len(children) != 1: # absent or ambiguous callable -> bail + continue + _emit_call(caller, children[0], rc) -def _cpp_preprocess(path: Path) -> bytes: - """Run cpp -w -P on a capital-F Fortran file and return preprocessed bytes. - Falls back to raw file bytes if cpp is not available. Capital-F extensions - conventionally require C preprocessor expansion (#ifdef MPI, #define REAL8, etc.) - before parsing. +def _resolve_typescript_member_calls( + per_file: list[dict], + all_nodes: list[dict], + all_edges: list[dict], +) -> None: + """Resolve cross-file TS/JS member calls via constructor-injection type tables (#1316). - Security (F-007): we pass `-nostdinc` and `-I /dev/null` so a malicious - source file containing `#include "/home/victim/.ssh/id_rsa"` (or any other - include directive) cannot inline arbitrary host files into the output that - we then ship to an LLM. Without these flags `cpp` happily resolves any - relative or absolute include path it can read, which is a corpus-side - file-exfiltration vector. + ``this.repo.findById()`` drops out in the shared cross-file pass because bare + ``findById`` collides across the corpus (god-node guard). TS constructors with + parameter-property modifiers (``private repo: IUserRepository``) produce a + per-file type table mapping field names to their declared types. This pass + looks up the receiver field's type, finds a single-definition class/interface + owning a method with the callee name, and emits an EXTRACTED ``calls`` edge. """ - import shutil - import subprocess - if not shutil.which("cpp"): - return path.read_bytes() - try: - # Pass an absolute path so a corpus file named like "-I/etc/x.F90" cannot - # be parsed by cpp as an option (cpp does not accept a "--" end-of-options - # terminator). An absolute path always begins with "/". - result = subprocess.run( - ["cpp", "-w", "-P", "-nostdinc", "-I", "/dev/null", str(path.resolve())], - capture_output=True, - timeout=30, - ) - if result.returncode == 0 and result.stdout: - return result.stdout - except Exception: - pass - return path.read_bytes() - - -def extract_fortran(path: Path) -> dict: - """Extract programs, modules, subroutines, functions, use statements, and calls from Fortran files. + type_table_by_file: dict[str, dict[str, str]] = {} + for result in per_file: + tt = result.get("ts_type_table") + if tt and tt.get("path"): + type_table_by_file[tt["path"]] = tt.get("table", {}) + if not type_table_by_file: + return - Capital-F extensions (.F, .F90, etc.) are run through the C preprocessor before - parsing so #ifdef/#define macros are resolved. - """ - try: - import tree_sitter_fortran as tsfortran - from tree_sitter import Language, Parser - except ImportError: - return {"nodes": [], "edges": [], "error": "tree-sitter-fortran not installed"} + def _key(label: str) -> str: + return re.sub(r"[^a-zA-Z0-9]+", "", str(label)).lower() - try: - language = Language(tsfortran.language()) - parser = Parser(language) - source = _cpp_preprocess(path) if path.suffix in _FORTRAN_CPP_EXTS else path.read_bytes() - tree = parser.parse(source) - root = tree.root_node - except Exception as e: - return {"nodes": [], "edges": [], "error": str(e)} + contained = {e.get("target") for e in all_edges if e.get("relation") == "contains"} - stem = _file_stem(path) - str_path = str(path) - nodes: list[dict] = [] - edges: list[dict] = [] - seen_ids: set[str] = set() - scope_bodies: list[tuple[str, object]] = [] + type_def_nids: dict[str, list[str]] = {} + node_by_id: dict[str, dict] = {} + for n in all_nodes: + node_by_id[n.get("id")] = n + if n.get("source_file") and n.get("id") in contained and _is_type_like_definition(n): + type_def_nids.setdefault(_key(n.get("label", "")), []).append(n["id"]) - def add_node(nid: str, label: str, line: int) -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": label, - "file_type": "code", - "source_file": str_path, - "source_location": f"L{line}", - }) + method_index: dict[tuple[str, str], str] = {} + for e in all_edges: + if e.get("relation") != "method": + continue + src, tgt = e.get("source"), e.get("target") + tnode = node_by_id.get(tgt) + if tnode is not None: + method_index[(src, _key(tnode.get("label", "")))] = tgt - def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", weight: float = 1.0, - context: str | None = None) -> None: - edge = { - "source": src, - "target": tgt, - "relation": relation, - "confidence": confidence, - "source_file": str_path, - "source_location": f"L{line}", - "weight": weight, - } - if context: - edge["context"] = context - edges.append(edge) + all_raw_calls: list[dict] = [] + for result in per_file: + all_raw_calls.extend(result.get("raw_calls", [])) - file_nid = _make_id(str(path)) - add_node(file_nid, path.name, 1) - - def _fortran_name(stmt_node) -> str | None: - """Extract name from a *_statement node. Fortran is case-insensitive; lowercase.""" - for child in stmt_node.children: - if child.type in ("name", "identifier"): - return _read_text(child, source).lower() - return None - - def ensure_named_node(name: str, line: int) -> str: - nid = _make_id(stem, name) - if nid in seen_ids: - return nid - nid = _make_id(name) - if nid not in seen_ids: - # The name isn't defined in this file, so this is a cross-file reference - # (e.g. a `Thing` type annotation imported from another module). Emit a - # SOURCELESS stub — like the inheritance-base path below — so the - # corpus-level rewire can collapse it onto the real definition. A sourced - # stub here makes _disambiguate_colliding_node_ids bake the referencing - # file's path (with extension) into the id and blocks the rewire, which is - # the phantom-duplicate-node bug (#1402). - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": name, - "file_type": "code", - "source_file": "", - "source_location": "", - "origin_file": str_path, - }) - return nid - - def emit_signature_refs(scope_node, fn_nid: str, is_function: bool) -> None: - """Emit references[parameter_type] / references[return_type] edges for - a subroutine/function based on its variable_declaration siblings.""" - stmt_type = "function_statement" if is_function else "subroutine_statement" - stmt = next((c for c in scope_node.children if c.type == stmt_type), None) - if stmt is None: - return - param_names: set[str] = set() - params_node = next((c for c in stmt.children if c.type == "parameters"), None) - if params_node is not None: - for c in params_node.children: - if c.type == "identifier": - param_names.add(_read_text(c, source).lower()) - result_name: str | None = None - if is_function: - result_node = next((c for c in stmt.children if c.type == "function_result"), None) - if result_node is not None: - res_id = next((c for c in result_node.children if c.type == "identifier"), None) - if res_id is not None: - result_name = _read_text(res_id, source).lower() - else: - # implicit result variable: same name as the function - result_name = _fortran_name(stmt) - for child in scope_node.children: - if child.type != "variable_declaration": - continue - derived = next((c for c in child.children if c.type == "derived_type"), None) - if derived is None: - continue - type_name_node = next((c for c in derived.children if c.type == "type_name"), None) - if type_name_node is None: - continue - type_name = _read_text(type_name_node, source).lower() - for var in child.children: - if var.type != "identifier": - continue - var_name = _read_text(var, source).lower() - var_line = var.start_point[0] + 1 - if var_name in param_names: - tgt = ensure_named_node(type_name, var_line) - if tgt != fn_nid: - add_edge(fn_nid, tgt, "references", var_line, context="parameter_type") - elif is_function and var_name == result_name: - tgt = ensure_named_node(type_name, var_line) - if tgt != fn_nid: - add_edge(fn_nid, tgt, "references", var_line, context="return_type") - - def walk_calls(node, scope_nid: str) -> None: - if node is None: - return - t = node.type - if t in ("subroutine", "function", "module", "program", "internal_procedures"): - return - # call FOO(args) — tree-sitter-fortran uses subroutine_call - if t == "subroutine_call": - name_node = next((c for c in node.children if c.type == "identifier"), None) - if name_node: - callee = _read_text(name_node, source).lower() - target_nid = _make_id(stem, callee) - add_edge(scope_nid, target_nid, "calls", node.start_point[0] + 1, - confidence="EXTRACTED", context="call") - # x = compute(args) — function invocations are `call_expression`, which - # shares Fortran's `name(...)` syntax with array indexing. Only emit a - # call edge when the callee resolves to a procedure defined in this file - # (an array variable produces no matching node), so array accesses can't - # fabricate spurious `calls` edges. - elif t == "call_expression": - name_node = next((c for c in node.children if c.type == "identifier"), None) - if name_node: - callee = _read_text(name_node, source).lower() - target_nid = _make_id(stem, callee) - if target_nid in seen_ids and target_nid != scope_nid: - add_edge(scope_nid, target_nid, "calls", node.start_point[0] + 1, - confidence="EXTRACTED", context="call") - for child in node.children: - walk_calls(child, scope_nid) - - def walk(node, scope_nid: str) -> None: - t = node.type - - if t == "program": - stmt = next((c for c in node.children if c.type == "program_statement"), None) - name = _fortran_name(stmt) if stmt else None - if name: - nid = _make_id(stem, name) - line = node.start_point[0] + 1 - add_node(nid, name, line) - add_edge(file_nid, nid, "defines", line) - scope_bodies.append((nid, node)) - for child in node.children: - walk(child, nid) - return - - if t == "module": - stmt = next((c for c in node.children if c.type == "module_statement"), None) - name = _fortran_name(stmt) if stmt else None - if name: - nid = _make_id(stem, name) - line = node.start_point[0] + 1 - add_node(nid, name, line) - add_edge(file_nid, nid, "defines", line) - for child in node.children: - walk(child, nid) - return - - # subroutines/functions inside a module live under internal_procedures - if t == "internal_procedures": - for child in node.children: - walk(child, scope_nid) - return - - if t == "derived_type_definition": - stmt = next((c for c in node.children if c.type == "derived_type_statement"), None) - if stmt is not None: - name_node = next((c for c in stmt.children if c.type == "type_name"), None) - if name_node is not None: - type_name = _read_text(name_node, source).lower() - type_nid = _make_id(stem, type_name) - line = node.start_point[0] + 1 - add_node(type_nid, type_name, line) - add_edge(scope_nid, type_nid, "defines", line) - return - - if t == "subroutine": - stmt = next((c for c in node.children if c.type == "subroutine_statement"), None) - name = _fortran_name(stmt) if stmt else None - if name: - nid = _make_id(stem, name) - line = node.start_point[0] + 1 - add_node(nid, f"{name}()", line) - add_edge(scope_nid, nid, "defines", line) - scope_bodies.append((nid, node)) - emit_signature_refs(node, nid, is_function=False) - for child in node.children: - walk(child, nid) - return - - if t == "function": - stmt = next((c for c in node.children if c.type == "function_statement"), None) - name = _fortran_name(stmt) if stmt else None - if name: - nid = _make_id(stem, name) - line = node.start_point[0] + 1 - add_node(nid, f"{name}()", line) - add_edge(scope_nid, nid, "defines", line) - scope_bodies.append((nid, node)) - emit_signature_refs(node, nid, is_function=True) - for child in node.children: - walk(child, nid) - return - - if t == "use_statement": - line = node.start_point[0] + 1 - # tree-sitter-fortran uses module_name node for the used module - name_node = next((c for c in node.children if c.type in ("module_name", "name", "identifier")), None) - if name_node: - mod_name = _read_text(name_node, source).lower() - imp_nid = _make_id(mod_name) - add_node(imp_nid, mod_name, line) - add_edge(scope_nid, imp_nid, "imports", line, context="use") - return - - for child in node.children: - walk(child, scope_nid) - - walk(root, file_nid) - - _stmt_headers = { - "subroutine_statement", "function_statement", - "program_statement", "module_statement", - } - for scope_nid, body_node in scope_bodies: - for child in body_node.children: - if child.type not in _stmt_headers: - walk_calls(child, scope_nid) - - return {"nodes": nodes, "edges": edges} - - -# ── Go extractor (custom walk) ──────────────────────────────────────────────── - -def extract_go(path: Path) -> dict: - """Extract functions, methods, type declarations, and imports from a .go file.""" - try: - import tree_sitter_go as tsgo - from tree_sitter import Language, Parser - except ImportError: - return {"nodes": [], "edges": [], "error": "tree-sitter-go not installed"} - - try: - language = Language(tsgo.language()) - parser = Parser(language) - source = path.read_bytes() - tree = parser.parse(source) - root = tree.root_node - except Exception as e: - return {"nodes": [], "edges": [], "error": str(e)} - - stem = _file_stem(path) - # Use directory name as package scope so methods on the same type across - # multiple files in a package share one canonical type node. - pkg_scope = path.parent.name or stem - str_path = str(path) - nodes: list[dict] = [] - edges: list[dict] = [] - seen_ids: set[str] = set() - function_bodies: list[tuple[str, object]] = [] - go_imported_pkgs: set[str] = set() # local names of imported packages - - def add_node(nid: str, label: str, line: int) -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": label, - "file_type": "code", - "source_file": str_path, - "source_location": f"L{line}", - }) - - def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", weight: float = 1.0, - context: str | None = None) -> None: - edge = { - "source": src, - "target": tgt, - "relation": relation, - "confidence": confidence, - "source_file": str_path, - "source_location": f"L{line}", - "weight": weight, - } - if context: - edge["context"] = context - edges.append(edge) - - file_nid = _make_id(str(path)) - add_node(file_nid, path.name, 1) - - def ensure_named_node(name: str, line: int) -> str: - nid = _make_id(pkg_scope, name) - if nid in seen_ids: - return nid - nid = _make_id(name) - if nid not in seen_ids: - # The name isn't declared in this file, so this is a cross-file reference - # (e.g. a type defined in another file of the package). Emit a SOURCELESS - # stub — like the inheritance-base path in the other extractors — so the - # corpus-level rewire can collapse it onto the real definition. A sourced - # stub here makes _disambiguate_colliding_node_ids bake the referencing - # file's path (with extension) into the id and blocks the rewire, which is - # the phantom-duplicate-node bug (#1402). - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": name, - "file_type": "code", - "source_file": "", - "source_location": "", - "origin_file": str_path, - }) - return nid - - def emit_go_method_refs(func_node, func_nid: str, line: int) -> None: - params = func_node.child_by_field_name("parameters") - if params is not None: - for p in params.children: - if p.type != "parameter_declaration": - continue - type_node = p.child_by_field_name("type") - refs: list[tuple[str, str]] = [] - _go_collect_type_refs(type_node, source, False, refs) - for ref_name, role in refs: - ctx = "generic_arg" if role == "generic_arg" else "parameter_type" - tgt = ensure_named_node(ref_name, line) - if tgt != func_nid: - add_edge(func_nid, tgt, "references", line, context=ctx) - result = func_node.child_by_field_name("result") - if result is not None: - if result.type == "parameter_list": - for p in result.children: - if p.type != "parameter_declaration": - continue - type_node = p.child_by_field_name("type") - if type_node is None: - for c in p.children: - if c.is_named: - type_node = c - break - refs = [] - _go_collect_type_refs(type_node, source, False, refs) - for ref_name, role in refs: - ctx = "generic_arg" if role == "generic_arg" else "return_type" - tgt = ensure_named_node(ref_name, line) - if tgt != func_nid: - add_edge(func_nid, tgt, "references", line, context=ctx) - else: - refs = [] - _go_collect_type_refs(result, source, False, refs) - for ref_name, role in refs: - ctx = "generic_arg" if role == "generic_arg" else "return_type" - tgt = ensure_named_node(ref_name, line) - if tgt != func_nid: - add_edge(func_nid, tgt, "references", line, context=ctx) - - def walk(node) -> None: - t = node.type - - if t == "function_declaration": - name_node = node.child_by_field_name("name") - if name_node: - func_name = _read_text(name_node, source) - line = node.start_point[0] + 1 - func_nid = _make_id(stem, func_name) - add_node(func_nid, f"{func_name}()", line) - add_edge(file_nid, func_nid, "contains", line) - emit_go_method_refs(node, func_nid, line) - body = node.child_by_field_name("body") - if body: - function_bodies.append((func_nid, body)) - return - - if t == "method_declaration": - receiver = node.child_by_field_name("receiver") - receiver_type: str | None = None - if receiver: - for param in receiver.children: - if param.type == "parameter_declaration": - type_node = param.child_by_field_name("type") - if type_node: - receiver_type = _read_text(type_node, source).lstrip("*").strip() - break - name_node = node.child_by_field_name("name") - if not name_node: - return - method_name = _read_text(name_node, source) - line = node.start_point[0] + 1 - - if receiver_type: - parent_nid = _make_id(pkg_scope, receiver_type) - add_node(parent_nid, receiver_type, line) - method_nid = _make_id(parent_nid, method_name) - add_node(method_nid, f".{method_name}()", line) - add_edge(parent_nid, method_nid, "method", line) - else: - method_nid = _make_id(stem, method_name) - add_node(method_nid, f"{method_name}()", line) - add_edge(file_nid, method_nid, "contains", line) - - emit_go_method_refs(node, method_nid, line) - body = node.child_by_field_name("body") - if body: - function_bodies.append((method_nid, body)) - return - - if t == "type_declaration": - for child in node.children: - if child.type != "type_spec": - continue - name_node = child.child_by_field_name("name") - if not name_node: - continue - type_name = _read_text(name_node, source) - line = child.start_point[0] + 1 - type_nid = _make_id(pkg_scope, type_name) - add_node(type_nid, type_name, line) - add_edge(file_nid, type_nid, "contains", line) - # Type body: struct fields (with embeds) or interface embedding. - type_body = None - for tc in child.children: - if tc.type in ("struct_type", "interface_type"): - type_body = tc - break - if type_body is None: - continue - if type_body.type == "struct_type": - for fdl in type_body.children: - if fdl.type != "field_declaration_list": - continue - for field in fdl.children: - if field.type != "field_declaration": - continue - has_name = any( - fc.type == "field_identifier" for fc in field.children - ) - type_node = field.child_by_field_name("type") - if type_node is None: - for fc in field.children: - if fc.is_named and fc.type != "field_identifier": - type_node = fc - break - refs: list[tuple[str, str]] = [] - _go_collect_type_refs(type_node, source, False, refs) - for ref_name, role in refs: - tgt = ensure_named_node(ref_name, field.start_point[0] + 1) - if tgt == type_nid: - continue - if not has_name and role == "type": - add_edge(type_nid, tgt, "embeds", - field.start_point[0] + 1) - else: - ctx = "generic_arg" if role == "generic_arg" else "field" - add_edge(type_nid, tgt, "references", - field.start_point[0] + 1, context=ctx) - elif type_body.type == "interface_type": - for elem in type_body.children: - if elem.type != "type_elem": - continue - refs = [] - for sub in elem.children: - if sub.is_named: - _go_collect_type_refs(sub, source, False, refs) - for ref_name, role in refs: - tgt = ensure_named_node(ref_name, elem.start_point[0] + 1) - if tgt == type_nid: - continue - if role == "type": - add_edge(type_nid, tgt, "embeds", - elem.start_point[0] + 1) - else: - add_edge(type_nid, tgt, "references", - elem.start_point[0] + 1, context="generic_arg") - return - - if t == "import_declaration": - for child in node.children: - if child.type == "import_spec_list": - for spec in child.children: - if spec.type == "import_spec": - path_node = spec.child_by_field_name("path") - if path_node: - raw = _read_text(path_node, source).strip('"') - # Prefix with go_pkg_ so stdlib names (e.g. "context") - # don't collide with local files of the same basename. - tgt_nid = _make_id("go", "pkg", raw) - add_edge(file_nid, tgt_nid, "imports_from", spec.start_point[0] + 1, context="import") - # Track local name (alias or last path segment) - alias = spec.child_by_field_name("name") - local_name = _read_text(alias, source) if alias else raw.split("/")[-1] - if local_name and local_name != "_" and local_name != ".": - go_imported_pkgs.add(local_name) - elif child.type == "import_spec": - path_node = child.child_by_field_name("path") - if path_node: - raw = _read_text(path_node, source).strip('"') - tgt_nid = _make_id("go", "pkg", raw) - add_edge(file_nid, tgt_nid, "imports_from", child.start_point[0] + 1, context="import") - alias = child.child_by_field_name("name") - local_name = _read_text(alias, source) if alias else raw.split("/")[-1] - if local_name and local_name != "_" and local_name != ".": - go_imported_pkgs.add(local_name) - return - - for child in node.children: - walk(child) - - walk(root) - - label_to_nid: dict[str, str] = {} - for n in nodes: - raw = n["label"] - normalised = raw.strip("()").lstrip(".") - label_to_nid[normalised] = n["id"] - - seen_call_pairs: set[tuple[str, str]] = set() - raw_calls: list[dict] = [] - - def walk_calls(node, caller_nid: str) -> None: - if node.type in ("function_declaration", "method_declaration"): - return - if node.type == "call_expression": - func_node = node.child_by_field_name("function") - callee_name: str | None = None - is_member_call: bool = False - if func_node: - if func_node.type == "identifier": - callee_name = _read_text(func_node, source) - elif func_node.type == "selector_expression": - field = func_node.child_by_field_name("field") - operand = func_node.child_by_field_name("operand") - receiver_name = _read_text(operand, source) if operand else "" - # Package-qualified call (e.g. fmt.Println) → allow cross-file resolution. - # Receiver method call (e.g. s.logger.Log) → skip, no import evidence. - is_member_call = receiver_name not in go_imported_pkgs - if field: - callee_name = _read_text(field, source) - if callee_name and callee_name not in _LANGUAGE_BUILTIN_GLOBALS: - tgt_nid = label_to_nid.get(callee_name) - if tgt_nid and tgt_nid != caller_nid: - pair = (caller_nid, tgt_nid) - if pair not in seen_call_pairs: - seen_call_pairs.add(pair) - line = node.start_point[0] + 1 - edges.append({ - "source": caller_nid, - "target": tgt_nid, - "relation": "calls", - "context": "call", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{line}", - "weight": 1.0, - }) - elif callee_name: - raw_calls.append({ - "caller_nid": caller_nid, - "callee": callee_name, - "is_member_call": is_member_call, - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - }) - for child in node.children: - walk_calls(child, caller_nid) - - for caller_nid, body_node in function_bodies: - walk_calls(body_node, caller_nid) - - valid_ids = seen_ids - clean_edges = [] - for edge in edges: - src, tgt = edge["source"], edge["target"] - if src in valid_ids and (tgt in valid_ids or edge["relation"] in ("imports", "imports_from")): - clean_edges.append(edge) - - return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls} - - -# ── Rust extractor (custom walk) ────────────────────────────────────────────── - -# Common Rust trait/stdlib method names that appear in virtually every codebase. -# Resolving these cross-file produces spurious INFERRED edges across crate -# boundaries (issue #908) — skip them from the unresolved-call queue entirely. -_RUST_TRAIT_METHOD_BLOCKLIST: frozenset[str] = frozenset({ - "new", "default", "parse", "from_str", "now", "clone", "into", "from", - "to_string", "to_owned", "len", "is_empty", "iter", "next", "build", - "start", "run", "init", "app", "get", "set", "push", "pop", "insert", - "remove", "contains", "collect", "map", "filter", "unwrap", "expect", - "ok", "err", "some", "none", "send", "recv", "lock", "read", "write", -}) - -def extract_rust(path: Path) -> dict: - """Extract functions, structs, enums, traits, impl methods, and use declarations from a .rs file.""" - try: - import tree_sitter_rust as tsrust - from tree_sitter import Language, Parser - except ImportError: - return {"nodes": [], "edges": [], "error": "tree-sitter-rust not installed"} - - try: - language = Language(tsrust.language()) - parser = Parser(language) - source = path.read_bytes() - tree = parser.parse(source) - root = tree.root_node - except Exception as e: - return {"nodes": [], "edges": [], "error": str(e)} - - stem = _file_stem(path) - str_path = str(path) - nodes: list[dict] = [] - edges: list[dict] = [] - seen_ids: set[str] = set() - function_bodies: list[tuple[str, object]] = [] - - def add_node(nid: str, label: str, line: int) -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": label, - "file_type": "code", - "source_file": str_path, - "source_location": f"L{line}", - }) - - def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", weight: float = 1.0, - context: str | None = None) -> None: - edge = { - "source": src, - "target": tgt, - "relation": relation, - "confidence": confidence, - "source_file": str_path, - "source_location": f"L{line}", - "weight": weight, - } - if context: - edge["context"] = context - edges.append(edge) - - file_nid = _make_id(str(path)) - add_node(file_nid, path.name, 1) - - def ensure_named_node(name: str, line: int) -> str: - nid = _make_id(stem, name) - if nid in seen_ids: - return nid - nid = _make_id(name) - if nid not in seen_ids: - # The name isn't defined in this file, so this is a cross-file reference - # (e.g. a `Thing` type annotation imported from another module). Emit a - # SOURCELESS stub — like the inheritance-base path below — so the - # corpus-level rewire can collapse it onto the real definition. A sourced - # stub here makes _disambiguate_colliding_node_ids bake the referencing - # file's path (with extension) into the id and blocks the rewire, which is - # the phantom-duplicate-node bug (#1402). - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": name, - "file_type": "code", - "source_file": "", - "source_location": "", - "origin_file": str_path, - }) - return nid - - def emit_param_return_refs(func_node, func_nid: str, line: int) -> None: - params = func_node.child_by_field_name("parameters") - if params is not None: - for p in params.children: - if p.type != "parameter": - continue - type_node = p.child_by_field_name("type") - refs: list[tuple[str, str]] = [] - _rust_collect_type_refs(type_node, source, False, refs) - for ref_name, role in refs: - ctx = "generic_arg" if role == "generic_arg" else "parameter_type" - tgt = ensure_named_node(ref_name, line) - if tgt != func_nid: - add_edge(func_nid, tgt, "references", line, context=ctx) - return_type = func_node.child_by_field_name("return_type") - if return_type is not None: - refs = [] - _rust_collect_type_refs(return_type, source, False, refs) - for ref_name, role in refs: - ctx = "generic_arg" if role == "generic_arg" else "return_type" - tgt = ensure_named_node(ref_name, line) - if tgt != func_nid: - add_edge(func_nid, tgt, "references", line, context=ctx) - - def walk(node, parent_impl_nid: str | None = None) -> None: - t = node.type - - if t == "function_item": - name_node = node.child_by_field_name("name") - if name_node: - func_name = _read_text(name_node, source) - line = node.start_point[0] + 1 - if parent_impl_nid: - func_nid = _make_id(parent_impl_nid, func_name) - add_node(func_nid, f".{func_name}()", line) - add_edge(parent_impl_nid, func_nid, "method", line) - else: - func_nid = _make_id(stem, func_name) - add_node(func_nid, f"{func_name}()", line) - add_edge(file_nid, func_nid, "contains", line) - emit_param_return_refs(node, func_nid, line) - body = node.child_by_field_name("body") - if body: - function_bodies.append((func_nid, body)) - return - - if t in ("struct_item", "enum_item", "trait_item"): - name_node = node.child_by_field_name("name") - if name_node: - item_name = _read_text(name_node, source) - line = node.start_point[0] + 1 - item_nid = _make_id(stem, item_name) - add_node(item_nid, item_name, line) - add_edge(file_nid, item_nid, "contains", line) - if t == "trait_item": - for c in node.children: - if c.type != "trait_bounds": - continue - for sub in c.children: - if not sub.is_named: - continue - refs: list[tuple[str, str]] = [] - _rust_collect_type_refs(sub, source, False, refs) - for idx, (ref_name, _role) in enumerate(refs): - tgt = ensure_named_node(ref_name, line) - if tgt == item_nid: - continue - rel = "inherits" if idx == 0 else "references" - if rel == "inherits": - add_edge(item_nid, tgt, "inherits", line) - else: - add_edge(item_nid, tgt, "references", line, - context="generic_arg") - if t == "struct_item": - for c in node.children: - if c.type != "field_declaration_list": - continue - for field in c.children: - if field.type != "field_declaration": - continue - type_node = field.child_by_field_name("type") - if type_node is None: - for fc in field.children: - if fc.type in ("type_identifier", "generic_type", - "scoped_type_identifier", - "reference_type", "primitive_type"): - type_node = fc - break - refs = [] - _rust_collect_type_refs(type_node, source, False, refs) - for ref_name, role in refs: - ctx = "generic_arg" if role == "generic_arg" else "field" - tgt = ensure_named_node(ref_name, field.start_point[0] + 1) - if tgt != item_nid: - add_edge(item_nid, tgt, "references", - field.start_point[0] + 1, context=ctx) - # Tuple structs (`struct Wrapper(pub Logger, Config);`) nest their - # positional field types directly under ordered_field_declaration_list - # with no field_declaration wrapper -- the same shape handled for tuple - # enum variants below. Without this branch these field type references - # are silently dropped. - for c in node.children: - if c.type != "ordered_field_declaration_list": - continue - fline = c.start_point[0] + 1 - for tc in c.children: - if tc.type not in ("type_identifier", "generic_type", - "scoped_type_identifier", "reference_type", - "primitive_type", "tuple_type", "array_type"): - continue - refs = [] - _rust_collect_type_refs(tc, source, False, refs) - for ref_name, role in refs: - ctx = "generic_arg" if role == "generic_arg" else "field" - tgt = ensure_named_node(ref_name, fline) - if tgt != item_nid: - add_edge(item_nid, tgt, "references", fline, context=ctx) - if t == "enum_item": - # Variant payload types nest under enum_variant_list -> - # enum_variant -> ordered_field_declaration_list (tuple variant, - # `Click(Logger)`) | field_declaration_list (struct variant, - # `Resize { size: Dim }`). Neither was traversed, so every - # enum-variant type reference was silently dropped. - _TYPE_NODES = ("type_identifier", "generic_type", - "scoped_type_identifier", "reference_type", - "primitive_type", "tuple_type", "array_type") - - def _emit_enum_type(type_node, at_line): - if type_node is None: - return - refs2: list[tuple[str, str]] = [] - _rust_collect_type_refs(type_node, source, False, refs2) - for ref_name, role in refs2: - ctx = "generic_arg" if role == "generic_arg" else "field" - tgt = ensure_named_node(ref_name, at_line) - if tgt != item_nid: - add_edge(item_nid, tgt, "references", at_line, context=ctx) - - for c in node.children: - if c.type != "enum_variant_list": - continue - for variant in c.children: - if variant.type != "enum_variant": - continue - vline = variant.start_point[0] + 1 - for vc in variant.children: - if vc.type == "ordered_field_declaration_list": - for tc in vc.children: - if tc.type in _TYPE_NODES: - _emit_enum_type(tc, vline) - elif vc.type == "field_declaration_list": - for field in vc.children: - if field.type != "field_declaration": - continue - type_node = field.child_by_field_name("type") - _emit_enum_type(type_node, field.start_point[0] + 1) - return - - if t == "impl_item": - type_node = node.child_by_field_name("type") - trait_node = node.child_by_field_name("trait") - impl_nid: str | None = None - if type_node: - type_name = _read_text(type_node, source).strip() - impl_nid = _make_id(stem, type_name) - add_node(impl_nid, type_name, node.start_point[0] + 1) - if trait_node is not None and impl_nid is not None: - refs: list[tuple[str, str]] = [] - _rust_collect_type_refs(trait_node, source, False, refs) - for idx, (ref_name, _role) in enumerate(refs): - tgt = ensure_named_node(ref_name, node.start_point[0] + 1) - if tgt == impl_nid: - continue - if idx == 0: - add_edge(impl_nid, tgt, "implements", node.start_point[0] + 1) - else: - add_edge(impl_nid, tgt, "references", node.start_point[0] + 1, - context="generic_arg") - body = node.child_by_field_name("body") - if body: - for child in body.children: - walk(child, parent_impl_nid=impl_nid) - return - - if t == "use_declaration": - arg = node.child_by_field_name("argument") - if arg: - raw = _read_text(arg, source) - clean = raw.split("{")[0].rstrip(":").rstrip("*").rstrip(":") - module_name = clean.split("::")[-1].strip() - if module_name: - tgt_nid = _make_id(module_name) - add_edge(file_nid, tgt_nid, "imports_from", node.start_point[0] + 1, context="import") - return - - for child in node.children: - walk(child, parent_impl_nid=None) - - walk(root) - - label_to_nid: dict[str, str] = {} - for n in nodes: - raw = n["label"] - normalised = raw.strip("()").lstrip(".") - label_to_nid[normalised] = n["id"] - - seen_call_pairs: set[tuple[str, str]] = set() - raw_calls: list[dict] = [] - - def walk_calls(node, caller_nid: str) -> None: - if node.type == "function_item": - return - if node.type == "call_expression": - func_node = node.child_by_field_name("function") - callee_name: str | None = None - is_member_call: bool = False - is_scoped_call: bool = False - if func_node: - if func_node.type == "identifier": - callee_name = _read_text(func_node, source) - elif func_node.type == "field_expression": - is_member_call = True - field = func_node.child_by_field_name("field") - if field: - callee_name = _read_text(field, source) - elif func_node.type == "scoped_identifier": - # Type::method() — still allow in-file EXTRACTED match, but - # skip cross-file resolution: bare last-segment lookup ignores - # crate boundaries and produces spurious INFERRED edges (#908). - is_scoped_call = True - name = func_node.child_by_field_name("name") - if name: - callee_name = _read_text(name, source) - if callee_name and callee_name not in _LANGUAGE_BUILTIN_GLOBALS: - tgt_nid = label_to_nid.get(callee_name) - if tgt_nid and tgt_nid != caller_nid: - pair = (caller_nid, tgt_nid) - if pair not in seen_call_pairs: - seen_call_pairs.add(pair) - line = node.start_point[0] + 1 - edges.append({ - "source": caller_nid, - "target": tgt_nid, - "relation": "calls", - "context": "call", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{line}", - "weight": 1.0, - }) - elif not is_scoped_call and callee_name.lower() not in _RUST_TRAIT_METHOD_BLOCKLIST: - raw_calls.append({ - "caller_nid": caller_nid, - "callee": callee_name, - "is_member_call": is_member_call, - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - }) - for child in node.children: - walk_calls(child, caller_nid) - - for caller_nid, body_node in function_bodies: - walk_calls(body_node, caller_nid) - - valid_ids = seen_ids - clean_edges = [] - for edge in edges: - src, tgt = edge["source"], edge["target"] - if src in valid_ids and (tgt in valid_ids or edge["relation"] in ("imports", "imports_from")): - clean_edges.append(edge) - - return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls} - - -# ── Zig ─────────────────────────────────────────────────────────────────────── - - - -# ── PowerShell ──────────────────────────────────────────────────────────────── - -def extract_powershell(path: Path) -> dict: - """Extract functions, classes, methods, and using statements from a .ps1 file.""" - try: - import tree_sitter_powershell as tsps - from tree_sitter import Language, Parser - except ImportError: - return {"nodes": [], "edges": [], "error": "tree_sitter_powershell not installed"} - - try: - language = Language(tsps.language()) - parser = Parser(language) - source = path.read_bytes() - tree = parser.parse(source) - root = tree.root_node - except Exception as e: - return {"nodes": [], "edges": [], "error": str(e)} - - stem = _file_stem(path) - str_path = str(path) - nodes: list[dict] = [] - edges: list[dict] = [] - seen_ids: set[str] = set() - function_bodies: list[tuple[str, Any]] = [] - - def add_node(nid: str, label: str, line: int) -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({"id": nid, "label": label, "file_type": "code", - "source_file": str_path, "source_location": f"L{line}"}) - - def add_edge(src: str, tgt: str, relation: str, line: int, - confidence: str = "EXTRACTED", weight: float = 1.0, - context: str | None = None) -> None: - edge = {"source": src, "target": tgt, "relation": relation, - "confidence": confidence, "source_file": str_path, - "source_location": f"L{line}", "weight": weight} - if context: - edge["context"] = context - edges.append(edge) - - file_nid = _make_id(str(path)) - add_node(file_nid, path.name, 1) - - _PS_SKIP = frozenset({ - "using", "return", "if", "else", "elseif", "foreach", "for", - "while", "do", "switch", "try", "catch", "finally", "throw", - "break", "continue", "exit", "param", "begin", "process", "end", - # Import commands — handled as import edges, not function calls - "import-module", - }) - - def _find_script_block_body(node): - for child in node.children: - if child.type == "script_block": - for sc in child.children: - if sc.type == "script_block_body": - return sc - return child - return None - - def ensure_named_node(name: str, line: int) -> str: - nid = _make_id(stem, name) - if nid in seen_ids: - return nid - nid = _make_id(name) - if nid not in seen_ids: - # The name isn't defined in this file, so this is a cross-file reference - # (e.g. a `Thing` type annotation imported from another module). Emit a - # SOURCELESS stub — like the inheritance-base path below — so the - # corpus-level rewire can collapse it onto the real definition. A sourced - # stub here makes _disambiguate_colliding_node_ids bake the referencing - # file's path (with extension) into the id and blocks the rewire, which is - # the phantom-duplicate-node bug (#1402). - seen_ids.add(nid) - nodes.append({ - "id": nid, - "label": name, - "file_type": "code", - "source_file": "", - "source_location": "", - "origin_file": str_path, - }) - return nid - - def _ps_type_name(type_literal_node) -> str | None: - """Drill into a type_literal node and return the inner type_identifier text.""" - if type_literal_node is None: - return None - for spec in type_literal_node.children: - if spec.type != "type_spec": - continue - for tname in spec.children: - if tname.type != "type_name": - continue - for tid in tname.children: - if tid.type == "type_identifier": - return _read_text(tid, source) - return None - - def walk(node, parent_class_nid: str | None = None) -> None: - t = node.type - - if t == "function_statement": - name_node = next((c for c in node.children if c.type == "function_name"), None) - if name_node: - func_name = _read_text(name_node, source) - line = node.start_point[0] + 1 - func_nid = _make_id(stem, func_name) - add_node(func_nid, f"{func_name}()", line) - add_edge(file_nid, func_nid, "contains", line) - body = _find_script_block_body(node) - if body: - function_bodies.append((func_nid, body)) - # Also walk the body during the main pass so that - # Import-Module / dot-source inside functions emit - # file-level imports_from edges (#1331). - walk(body, parent_class_nid) - return - - if t == "class_statement": - name_node = next((c for c in node.children if c.type == "simple_name"), None) - if name_node: - class_name = _read_text(name_node, source) - line = node.start_point[0] + 1 - class_nid = _make_id(stem, class_name) - add_node(class_nid, class_name, line) - add_edge(file_nid, class_nid, "contains", line) - # Base type(s) after ':'. PowerShell has no syntactic base vs - # interface split, so (matching the C# convention) treat the - # first base as the superclass (inherits) and the rest as - # interfaces (implements). Bases are the simple_name children - # after the ':' token. - colon_seen = False - base_index = 0 - for child in node.children: - if child.type == ":": - colon_seen = True - elif colon_seen and child.type == "simple_name": - base_nid = ensure_named_node(_read_text(child, source), line) - if base_nid != class_nid: - rel = "inherits" if base_index == 0 else "implements" - add_edge(class_nid, base_nid, rel, line) - base_index += 1 - for child in node.children: - walk(child, parent_class_nid=class_nid) - return - - if t == "class_property_definition" and parent_class_nid: - type_literal = next((c for c in node.children if c.type == "type_literal"), None) - type_name = _ps_type_name(type_literal) - if type_name: - line = node.start_point[0] + 1 - target_nid = ensure_named_node(type_name, line) - if target_nid != parent_class_nid: - add_edge(parent_class_nid, target_nid, "references", - line, context="field") - return - - if t == "class_method_definition": - name_node = next((c for c in node.children if c.type == "simple_name"), None) - if name_node: - method_name = _read_text(name_node, source) - line = node.start_point[0] + 1 - if parent_class_nid: - method_nid = _make_id(parent_class_nid, method_name) - add_node(method_nid, f".{method_name}()", line) - add_edge(parent_class_nid, method_nid, "method", line) - else: - method_nid = _make_id(stem, method_name) - add_node(method_nid, f"{method_name}()", line) - add_edge(file_nid, method_nid, "contains", line) - # Return type: type_literal sibling of simple_name - return_type_literal = next( - (c for c in node.children if c.type == "type_literal"), None) - return_type_name = _ps_type_name(return_type_literal) - if return_type_name: - target_nid = ensure_named_node(return_type_name, line) - if target_nid != method_nid: - add_edge(method_nid, target_nid, "references", - line, context="return_type") - # Parameter types: class_method_parameter_list - param_list = next( - (c for c in node.children if c.type == "class_method_parameter_list"), None) - if param_list is not None: - for p in param_list.children: - if p.type != "class_method_parameter": - continue - ptype_literal = next( - (c for c in p.children if c.type == "type_literal"), None) - ptype_name = _ps_type_name(ptype_literal) - if not ptype_name: - continue - p_line = p.start_point[0] + 1 - target_nid = ensure_named_node(ptype_name, p_line) - if target_nid != method_nid: - add_edge(method_nid, target_nid, "references", - p_line, context="parameter_type") - body = _find_script_block_body(node) - if body: - function_bodies.append((method_nid, body)) - return - - if t == "command": - # Dot-sourcing: `. ./Shared.psm1` - # Uses command_invokation_operator '.' + command_name_expr (not command_name) - invoke_op = next( - (c for c in node.children if c.type == "command_invokation_operator"), None - ) - if invoke_op is not None and _read_text(invoke_op, source).strip() == ".": - name_expr = next( - (c for c in node.children if c.type == "command_name_expr"), None - ) - if name_expr is not None: - name_node = next( - (c for c in name_expr.children if c.type == "command_name"), None - ) - if name_node: - raw_path = _read_text(name_node, source) - # Strip relative path prefix (./ or .\ or just the dot) - module_stem = re.sub(r'^[./\\]+', '', raw_path) - # Drop extension to get bare module name - module_stem = re.sub(r'\.[^.]+$', '', module_stem).replace('\\', '/') - module_name = module_stem.split('/')[-1] - if module_name: - add_edge(file_nid, _make_id(module_name), "imports_from", - node.start_point[0] + 1) - return - - cmd_name_node = next((c for c in node.children if c.type == "command_name"), None) - if cmd_name_node: - cmd_text = _read_text(cmd_name_node, source).lower() - if cmd_text == "using": - tokens = [] - for child in node.children: - if child.type == "command_elements": - for el in child.children: - if el.type == "generic_token": - tokens.append(_read_text(el, source)) - module_tokens = [t for t in tokens - if t.lower() not in ("namespace", "module", "assembly")] - if module_tokens: - module_name = module_tokens[-1].split(".")[-1] - add_edge(file_nid, _make_id(module_name), "imports_from", - node.start_point[0] + 1) - elif cmd_text == "import-module": - # Collect generic_token args; skip command_parameter flags like -Name - # The module name is the first generic_token (or the one after -Name) - module_name: str | None = None - expect_name = False - for child in node.children: - if child.type != "command_elements": - continue - for el in child.children: - if el.type == "command_parameter": - param_text = _read_text(el, source).lstrip("-").lower() - expect_name = param_text in ("name", "n") - elif el.type == "generic_token": - token = _read_text(el, source) - if module_name is None or expect_name: - module_name = token - expect_name = False - if module_name: - # Strip extension; keep only the stem for the node ID - bare = re.sub(r'\.[^.]+$', '', module_name).split('/')[-1].split('\\')[-1] - if bare: - add_edge(file_nid, _make_id(bare), "imports_from", - node.start_point[0] + 1) - return - - for child in node.children: - walk(child, parent_class_nid) - - walk(root) - - label_to_nid = {n["label"].strip("()").lstrip(".").lower(): n["id"] for n in nodes} - seen_call_pairs: set[tuple[str, str]] = set() - raw_calls: list[dict] = [] - - def walk_calls(node, caller_nid: str) -> None: - if node.type in ("function_statement", "class_statement"): - return - if node.type == "command": - cmd_name_node = next((c for c in node.children if c.type == "command_name"), None) - if cmd_name_node: - cmd_text = _read_text(cmd_name_node, source) - if cmd_text.lower() not in _PS_SKIP: - tgt_nid = label_to_nid.get(cmd_text.lower()) - if tgt_nid and tgt_nid != caller_nid: - pair = (caller_nid, tgt_nid) - if pair not in seen_call_pairs: - seen_call_pairs.add(pair) - add_edge(caller_nid, tgt_nid, "calls", - node.start_point[0] + 1, - confidence="EXTRACTED", weight=1.0) - elif cmd_text: - raw_calls.append({ - "caller_nid": caller_nid, - "callee": cmd_text, - "is_member_call": False, - "source_file": str_path, - "source_location": f"L{node.start_point[0] + 1}", - }) - for child in node.children: - walk_calls(child, caller_nid) - - for caller_nid, body_node in function_bodies: - walk_calls(body_node, caller_nid) - - clean_edges = [e for e in edges if e["source"] in seen_ids and - (e["target"] in seen_ids or e["relation"] in ("imports_from", "imports"))] - return {"nodes": nodes, "edges": clean_edges, "raw_calls": raw_calls} - - -# ── PowerShell manifest (.psd1) ────────────────────────────────────────────── - -# Keys in a .psd1 whose values are module names/paths we treat as imports. -_PSD1_IMPORT_KEYS = frozenset({"RootModule", "NestedModules", "RequiredModules"}) - - -def _psd1_collect_string_literals(node, source: bytes) -> list[str]: - """Recursively collect all string_literal text values under *node*.""" - results: list[str] = [] - - def _walk(n) -> None: - if n.type == "string_literal": - raw = source[n.start_byte:n.end_byte].decode(errors="replace") - # Strip surrounding quote chars (' or ") - results.append(raw.strip("'\"")) - return - for child in n.children: - _walk(child) - - _walk(node) - return results - - -def _psd1_module_name(raw: str) -> str: - """Derive a bare module name from a raw string value. - - e.g. 'MyModule.psm1' → 'MyModule', './sub/Util.psm1' → 'Util', 'PSReadLine' → 'PSReadLine' - """ - # Strip path prefix and extension - name = raw.replace("\\", "/").split("/")[-1] - name = re.sub(r"\.[^.]+$", "", name) # remove last extension - return name.strip() - - -def extract_powershell_manifest(path: Path) -> dict: - """Extract module dependency edges from a PowerShell .psd1 manifest file. - - .psd1 files are PowerShell data hashtables, not scripts. tree-sitter-powershell - parses them correctly (they are syntactically valid PS). We walk the AST looking - for RootModule, NestedModules, and RequiredModules keys and emit imports_from - edges for every referenced module. - - RequiredModules supports two forms: - - Simple string: 'PSReadLine' - - Module specification: @{ ModuleName = 'Pester'; ModuleVersion = '5.0' } - For the hashtable form we only follow the ModuleName key. - """ - try: - import tree_sitter_powershell as tsps - from tree_sitter import Language, Parser - except ImportError: - return {"nodes": [], "edges": [], "error": "tree_sitter_powershell not installed"} - - try: - language = Language(tsps.language()) - parser = Parser(language) - source = path.read_bytes() - tree = parser.parse(source) - root = tree.root_node - except Exception as e: - return {"nodes": [], "edges": [], "error": str(e)} - - str_path = str(path) - nodes: list[dict] = [] - edges: list[dict] = [] - seen_ids: set[str] = set() - - def add_node(nid: str, label: str, line: int) -> None: - if nid not in seen_ids: - seen_ids.add(nid) - nodes.append({"id": nid, "label": label, "file_type": "code", - "source_file": str_path, "source_location": f"L{line}"}) - - def add_import_edge(src: str, module_raw: str, line: int) -> None: - name = _psd1_module_name(module_raw) - if not name: - return - tgt_nid = _make_id(name) - edges.append({ - "source": src, - "target": tgt_nid, - "relation": "imports_from", - "confidence": "EXTRACTED", - "source_file": str_path, - "source_location": f"L{line}", - "weight": 1.0, - "context": "import", - }) - - file_nid = _make_id(str(path)) - add_node(file_nid, path.name, 1) - - def walk_manifest(node) -> None: - """Walk the AST and emit edges for import-relevant hash_entry nodes.""" - if node.type != "hash_entry": - for child in node.children: - walk_manifest(child) - return - - # Identify the key - key_node = next((c for c in node.children if c.type == "key_expression"), None) - if key_node is None: - return - key_text = source[key_node.start_byte:key_node.end_byte].decode(errors="replace").strip() - - if key_text not in _PSD1_IMPORT_KEYS: - # Still recurse in case there are nested hashes (e.g. ModuleVersion entries - # contain sub-hashes, but we only care about top-level keys for imports) - return - - line = node.start_point[0] + 1 - value_node = next((c for c in node.children if c.type == "pipeline"), None) - if value_node is None: - return - - if key_text == "RootModule": - # Value is a single string - strings = _psd1_collect_string_literals(value_node, source) - for s in strings: - add_import_edge(file_nid, s, line) - - elif key_text == "NestedModules": - # Value is a string or @('a', 'b', ...) array — collect all string literals - strings = _psd1_collect_string_literals(value_node, source) - for s in strings: - add_import_edge(file_nid, s, line) - - elif key_text == "RequiredModules": - # Two forms: - # 1) 'SimpleModule' — direct string literals in the array - # 2) @{ ModuleName = 'Foo'; ModuleVersion = '2.0' } — use ModuleName only - # - # Strategy: walk the value for hash_entry nodes whose key is 'ModuleName'; - # collect their string values. For the remaining string_literal nodes that - # are NOT inside a hash_entry subtree, treat them as simple module names. - module_name_strings: list[str] = [] - inside_hash_entries: set[int] = set() # byte offsets of handled strings - - def find_modulename_entries(n) -> None: - if n.type == "hash_entry": - sub_key = next((c for c in n.children if c.type == "key_expression"), None) - if sub_key is not None: - sk_text = source[sub_key.start_byte:sub_key.end_byte].decode(errors="replace").strip() - # Collect strings inside *all* sub-keys so we can exclude them - for c in n.children: - if c.type == "pipeline": - for s_node in _collect_string_nodes(c): - inside_hash_entries.add(s_node.start_byte) - if sk_text == "ModuleName": - for c in n.children: - if c.type == "pipeline": - for s in _psd1_collect_string_literals(c, source): - module_name_strings.append(s) - return # don't recurse further into this hash_entry - for child in n.children: - find_modulename_entries(child) - - def _collect_string_nodes(n): - """Return all string_literal nodes in subtree.""" - if n.type == "string_literal": - yield n - return - for child in n.children: - yield from _collect_string_nodes(child) - - find_modulename_entries(value_node) - - # Now gather direct string literals not inside hash entries - direct_strings: list[str] = [] - for s_node in _collect_string_nodes(value_node): - if s_node.start_byte not in inside_hash_entries: - raw = source[s_node.start_byte:s_node.end_byte].decode(errors="replace") - direct_strings.append(raw.strip("'\"")) - - for s in direct_strings + module_name_strings: - add_import_edge(file_nid, s, line) - - walk_manifest(root) - - return {"nodes": nodes, "edges": edges, "raw_calls": []} - - -# ── Cross-file import resolution ────────────────────────────────────────────── - -def _source_key(source_file: str, root: Path) -> str: - if not source_file: - return "" - source_path = Path(source_file) - try: - return str(source_path.resolve().relative_to(root)) - except Exception: - return str(source_path) - - -def _node_disambiguation_source_key(node: dict, root: Path) -> str: - source_file = str(node.get("source_file", "")) - if source_file: - return _source_key(source_file, root) - return _source_key(str(node.get("origin_file", "")), root) - - -def _disambiguate_colliding_node_ids( - nodes: list[dict], - edges: list[dict], - raw_calls: list[dict], - root: Path, -) -> None: - """Rewrite only colliding node IDs, using source path as the disambiguator. - - Module anchor nodes (#1327) are exempt: ``import CoreKit`` from three files - yields three ``type=module`` nodes with the same id but different - source_files. Those are the *same* module, not distinct same-named symbols, - so they must collapse to one shared node — disambiguating them by path would - scatter a single module across N file-qualified duplicates. - """ - by_id: dict[str, list[dict]] = {} - for node in nodes: - if node.get("type") in ("module", "namespace"): - continue - nid = node.get("id") - if isinstance(nid, str) and nid: - by_id.setdefault(nid, []).append(node) - - remap: dict[tuple[str, str], str] = {} - ambiguous_ids: set[str] = set() - for old_id, group in by_id.items(): - source_keys = {_node_disambiguation_source_key(node, root) for node in group} - if len(group) < 2 or len(source_keys) < 2: - continue - ambiguous_ids.add(old_id) - # Salt the colliding id with the *path* it came from. The naive salt is - # ``_make_id(source_key, old_id)`` — source_key is the raw repo-relative - # path. But _make_id collapses every separator, so two DISTINCT paths - # whose only difference is a separator-vs-inner-punctuation swap - # (``a/b/c.md`` vs ``a.b/c.md``, ``foo/bar_baz.md`` vs ``foo_bar/baz.md``) - # normalize to the SAME salted id and still collide (#1522 — the residual - # of #1504 the 0.9.0 full-path stem didn't reach). When that happens, - # append a short stable hash of the *raw* source_key, which IS injective - # over distinct paths, so the colliders separate. Computed in code from - # source_file (never trusted from the LLM), so AST↔semantic parity holds. - naive: dict[str, str] = {} # source_key -> _make_id(source_key, old_id) - for source_key in source_keys: - if source_key: - naive[source_key] = _make_id(source_key, old_id) - # source_keys that, after normalization, are not unique among themselves. - seen: dict[str, int] = {} - for nid in naive.values(): - seen[nid] = seen.get(nid, 0) + 1 - needs_hash = {sk for sk, nid in naive.items() if seen.get(nid, 0) > 1} - for node in group: - source_key = _node_disambiguation_source_key(node, root) - if not source_key: - continue - if source_key in needs_hash: - salt = hashlib.sha1(source_key.encode("utf-8")).hexdigest()[:6] - new_id = _make_id(source_key, old_id, salt) - else: - new_id = naive.get(source_key) or _make_id(source_key, old_id) - remap[(old_id, source_key)] = new_id - if new_id != old_id: - node["id"] = new_id - - if not remap: - return - - unambiguous_remaps: dict[str, str] = {} - for old_id, group in by_id.items(): - if old_id in ambiguous_ids: - continue - candidates = { - node["id"] for node in group - if isinstance(node.get("id"), str) and node["id"] != old_id - } - if len(candidates) == 1: - unambiguous_remaps[old_id] = next(iter(candidates)) - - # A C/ObjC/C++ `#include "foo.h"` / `#import "foo.h"` resolves to the header's - # file node, but `foo.h` and its sibling `foo.c`/`foo.m`/`foo.cpp` collapse to - # the same `foo` file id, so disambiguation salts them apart by path. A - # cross-file import edge from a THIRD file carries neither salt's source_key, so - # the (target, edge_source_key) lookup misses and the edge dangles on the now - # dead `foo` id. Repoint those import edges to the HEADER variant (the include - # always targeted the header), keyed by the original colliding id (#1475). - _HEADER_SUFFIXES = (".h", ".hpp", ".hh", ".hxx") - header_remaps: dict[str, str] = {} - for old_id in ambiguous_ids: - for node in by_id.get(old_id, []): - sk = _node_disambiguation_source_key(node, root) - if sk and Path(sk).suffix.lower() in _HEADER_SUFFIXES: - new_id = remap.get((old_id, sk)) - if new_id: - header_remaps[old_id] = new_id - break - - for edge in edges: - edge_source_key = _source_key(str(edge.get("source_file", "")), root) - source_key = (edge.get("source", ""), edge_source_key) - target_key = (edge.get("target", ""), edge_source_key) - if source_key in remap: - edge["source"] = remap[source_key] - elif edge.get("source") in unambiguous_remaps: - edge["source"] = unambiguous_remaps[str(edge["source"])] - # imports/imports_from always target a header file, so they must resolve to - # the header variant BEFORE the same-source-file salt is considered. Keying - # the import target by the importer's own source file mis-points a `.m` - # importing its own `.h` back at itself (self-loop), and is wrong for any - # cross-file import whose importer shares the colliding id (#1475). - if (edge.get("relation") in ("imports", "imports_from") - and edge.get("target") in header_remaps): - edge["target"] = header_remaps[str(edge["target"])] - elif target_key in remap: - edge["target"] = remap[target_key] - elif edge.get("target") in unambiguous_remaps: - edge["target"] = unambiguous_remaps[str(edge["target"])] - - for raw_call in raw_calls: - call_source_key = _source_key(str(raw_call.get("source_file", "")), root) - caller_key = (raw_call.get("caller_nid", ""), call_source_key) - if caller_key in remap: - raw_call["caller_nid"] = remap[caller_key] - elif raw_call.get("caller_nid") in unambiguous_remaps: - raw_call["caller_nid"] = unambiguous_remaps[str(raw_call["caller_nid"])] - - -def _canonicalize_csharp_namespace_nodes(all_nodes: list[dict], all_edges: list[dict]) -> None: - """Collapse duplicate C# namespace node entries to one canonical node per label.""" - by_label: dict[str, list[dict]] = {} - for node in all_nodes: - if node.get("type") != "namespace": - continue - label = node.get("label") - if isinstance(label, str): - by_label.setdefault(label, []).append(node) - - remap: dict[str, str] = {} - drop_node_ids: set[int] = set() - for group in by_label.values(): - if len(group) < 2: - continue - canonical = sorted( - group, - key=lambda node: ( - str(node.get("source_file") or ""), - str(node.get("source_location") or ""), - str(node.get("id") or ""), - ), - )[0] - canonical_id = canonical.get("id") - for node in group: - if node is canonical: - continue - drop_node_ids.add(id(node)) - dup_id = node.get("id") - if isinstance(dup_id, str) and isinstance(canonical_id, str): - remap[dup_id] = canonical_id - - if remap: - for edge in all_edges: - if edge.get("source") in remap: - edge["source"] = remap[str(edge["source"])] - if edge.get("target") in remap: - edge["target"] = remap[str(edge["target"])] - - if drop_node_ids: - all_nodes[:] = [node for node in all_nodes if id(node) not in drop_node_ids] - - -# Languages whose identifiers are case-insensitive, so cross-file name resolution -# may fold case. Everywhere else, case is semantic (`Path` the class vs `PATH` the -# env var are distinct) and folding manufactures false edges / super-hubs (#1581). -_CASE_INSENSITIVE_EXTS = frozenset({ - ".php", ".phtml", ".php3", ".php4", ".php5", ".php7", ".phps", # PHP fns/classes - ".sql", # SQL identifiers - ".nim", ".nims", ".nimble", # Nim (style-insensitive) -}) - - -def _lang_is_case_insensitive(source_file: object) -> bool: - """True when the file's language resolves identifiers case-insensitively (#1581).""" - if not source_file: - return False - return Path(str(source_file)).suffix.lower() in _CASE_INSENSITIVE_EXTS - - -def _node_label_key(node: dict, fold: bool = False) -> str: - label = str(node.get("label", "")).strip() - key = re.sub(r"[^a-zA-Z0-9]+", "", label) - return key.lower() if fold else key - - -def _is_type_like_definition(node: dict) -> bool: - if node.get("type") == "namespace": - return False - label = str(node.get("label", "")).strip() - if not label: - return False - if label.endswith(")") or label.startswith("."): - return False - if "." in label: - return False - return node.get("file_type") == "code" - - -def _rewire_unique_stub_nodes(nodes: list[dict], edges: list[dict]) -> None: - """Map unresolved no-source stubs to a unique real definition with the same label.""" - real_by_label: dict[str, list[dict]] = {} # exact-case (all languages) - real_by_label_ci: dict[str, list[dict]] = {} # case-INSENSITIVE-language reals only - stubs: list[dict] = [] - - for node in nodes: - key = _node_label_key(node) - if not key: - continue - if node.get("source_file"): - if _is_type_like_definition(node): - # Match stubs case-SENSITIVELY: a `Path` reference must not rewire to a - # `PATH` env var (#1581). Fold only for genuinely case-insensitive - # languages, where `foo` legitimately resolves to `Foo`. - real_by_label.setdefault(key, []).append(node) - if _lang_is_case_insensitive(node.get("source_file")): - real_by_label_ci.setdefault( - _node_label_key(node, fold=True), []).append(node) - continue - stubs.append(node) - - remap: dict[str, str] = {} - for stub in stubs: - stub_id = str(stub.get("id", "")) - if not stub_id: - continue - candidates = real_by_label.get(_node_label_key(stub), []) - if len(candidates) != 1: - # No unique exact match — fall back to a case-insensitive match, but - # only against case-insensitive-language definitions (so a case-sensitive - # `PATH` can never absorb a `Path` reference). - candidates = real_by_label_ci.get(_node_label_key(stub, fold=True), []) - if len(candidates) != 1: - continue - target_id = candidates[0].get("id") - if isinstance(target_id, str) and target_id and target_id != stub_id: - remap[stub_id] = target_id - - if not remap: - return - - by_id = {node.get("id"): node for node in nodes if node.get("id")} - csharp_scoped_relations = {"inherits", "implements", "references", "imports"} - for edge in edges: - is_csharp_scoped_edge = ( - str(edge.get("source_file", "")).endswith(".cs") - and edge.get("relation") in csharp_scoped_relations - ) - source = edge.get("source") - if source in remap: - remapped_source = remap[str(source)] - if not ( - is_csharp_scoped_edge - and str(by_id.get(remapped_source, {}).get("source_file", "")).endswith(".cs") - ): - edge["source"] = remapped_source - target = edge.get("target") - if target in remap: - remapped_target = remap[str(target)] - if not ( - is_csharp_scoped_edge - and str(by_id.get(remapped_target, {}).get("source_file", "")).endswith(".cs") - ): - edge["target"] = remapped_target - - referenced = {x for e in edges for x in (e.get("source"), e.get("target"))} - drop_ids = {stub_id for stub_id in remap if stub_id not in referenced} - nodes[:] = [node for node in nodes if node.get("id") not in drop_ids] - - -def _js_source_path(source_file: str, root: Path) -> Path | None: - if not source_file: - return None - path = Path(source_file) - if not path.is_absolute(): - path = root / path - try: - return path.resolve() - except Exception: - return path - - -@dataclass(frozen=True) -class _SymbolDeclarationFact: - file_path: Path - name: str - line: int - - -@dataclass(frozen=True) -class _SymbolImportFact: - file_path: Path - local_name: str - target_path: Path - imported_name: str - line: int - - -@dataclass(frozen=True) -class _SymbolAliasFact: - file_path: Path - alias: str - target_name: str - line: int - - -@dataclass(frozen=True) -class _SymbolExportFact: - file_path: Path - exported_name: str - line: int - local_name: str | None = None - target_path: Path | None = None - target_name: str | None = None - - -@dataclass(frozen=True) -class _StarExportFact: - file_path: Path - target_path: Path - line: int - - -@dataclass(frozen=True) -class _NamespaceExportFact: - file_path: Path - exported_name: str - target_path: Path - line: int - - -@dataclass(frozen=True) -class _SymbolUseFact: - file_path: Path - source_id: str - local_name: str - relation: str - context: str - line: int - - -@dataclass -class _SymbolResolutionFacts: - declarations: list[_SymbolDeclarationFact] = field(default_factory=list) - imports: list[_SymbolImportFact] = field(default_factory=list) - aliases: list[_SymbolAliasFact] = field(default_factory=list) - exports: list[_SymbolExportFact] = field(default_factory=list) - star_exports: list[_StarExportFact] = field(default_factory=list) - namespace_exports: list[_NamespaceExportFact] = field(default_factory=list) - uses: list[_SymbolUseFact] = field(default_factory=list) - # File-to-file submodule imports from `from pkg import submod` (#1146). - # Each entry is (importing_file, submodule_file, line). - module_imports: list[tuple[Path, Path, int]] = field(default_factory=list) - - -def _apply_symbol_resolution_facts( - paths: list[Path], - nodes: list[dict], - edges: list[dict], - root: Path, - facts: _SymbolResolutionFacts, -) -> None: - """Apply language-provided import/export/use facts to graph edges.""" - if not ( - facts.declarations - or facts.imports - or facts.aliases - or facts.exports - or facts.star_exports - or facts.namespace_exports - or facts.uses - or facts.module_imports - ): - return - - path_by_resolved = {path.resolve(): path for path in paths} - source_file_id = {path.resolve(): _make_id(str(path)) for path in paths} - symbol_nodes: dict[tuple[Path, str], str] = {} - for node in nodes: - source_path = _js_source_path(str(node.get("source_file", "")), root) - if source_path is None: - continue - label = str(node.get("label", "")).strip().strip("()").lstrip(".") - if label and node.get("id"): - symbol_nodes[(source_path, label)] = str(node["id"]) - - def ensure_symbol_node(path: Path, name: str, line: int) -> str: - resolved_path = path.resolve() - existing = symbol_nodes.get((resolved_path, name)) - if existing is not None: - return existing - node_id = _make_id(_file_stem(path), name) - symbol_nodes[(resolved_path, name)] = node_id - nodes.append({ - "id": node_id, - "label": name, - "file_type": "code", - "source_file": str(path), - "source_location": f"L{line}", - }) - return node_id - - existing_edges = { - ( - str(edge.get("source")), - str(edge.get("target")), - str(edge.get("relation")), - str(edge.get("context") or ""), - ) - for edge in edges - } - - def add_edge(source: str, target: str, relation: str, context: str, line: int, source_path: Path) -> None: - key = (source, target, relation, context or "") - if key in existing_edges: - return - existing_edges.add(key) - edges.append({ - "source": source, - "target": target, - "relation": relation, - "context": context, - "confidence": "EXTRACTED", - "source_file": str(source_path), - "source_location": f"L{line}", - "weight": 1.0, - }) - - for declaration in facts.declarations: - ensure_symbol_node(declaration.file_path, declaration.name, declaration.line) - - local_aliases_by_file: dict[Path, dict[str, tuple[Path, str]]] = {} - for import_fact in facts.imports: - file_path = import_fact.file_path.resolve() - local_aliases_by_file.setdefault(file_path, {})[import_fact.local_name] = ( - import_fact.target_path.resolve(), - import_fact.imported_name, - ) - - pending_aliases_by_file: dict[Path, list[_SymbolAliasFact]] = {} - for alias_fact in facts.aliases: - pending_aliases_by_file.setdefault(alias_fact.file_path.resolve(), []).append(alias_fact) - - for file_path, aliases in pending_aliases_by_file.items(): - local_aliases = local_aliases_by_file.setdefault(file_path, {}) - changed = True - while changed: - changed = False - for alias_fact in aliases: - if alias_fact.alias in local_aliases: - continue - origin = local_aliases.get(alias_fact.target_name) - if origin is not None: - local_aliases[alias_fact.alias] = origin - changed = True - - named_exports_by_file: dict[Path, dict[str, tuple[Path, str]]] = {} - star_exports_by_file: dict[Path, list[Path]] = {} - - for star_fact in facts.star_exports: - source_path = star_fact.file_path.resolve() - target_path = star_fact.target_path.resolve() - star_exports_by_file.setdefault(source_path, []).append(target_path) - source_id = source_file_id.get(source_path) - if source_id is not None: - add_edge( - source_id, - _make_id(str(path_by_resolved.get(target_path, target_path))), - "re_exports", - "export", - star_fact.line, - star_fact.file_path, - ) - - for namespace_fact in facts.namespace_exports: - source_path = namespace_fact.file_path.resolve() - target_path = namespace_fact.target_path.resolve() - namespace_id = ensure_symbol_node( - namespace_fact.file_path, - namespace_fact.exported_name, - namespace_fact.line, - ) - named_exports_by_file.setdefault(source_path, {})[ - namespace_fact.exported_name - ] = (source_path, namespace_fact.exported_name) - source_id = source_file_id.get(source_path) - if source_id is not None: - add_edge( - source_id, - namespace_id, - "contains", - "namespace_export", - namespace_fact.line, - namespace_fact.file_path, - ) - add_edge( - source_id, - _make_id(str(path_by_resolved.get(target_path, target_path))), - "re_exports", - "export", - namespace_fact.line, - namespace_fact.file_path, - ) - - for export_fact in facts.exports: - file_path = export_fact.file_path.resolve() - origin: tuple[Path, str] | None = None - if export_fact.target_path is not None and export_fact.target_name is not None: - origin = (export_fact.target_path.resolve(), export_fact.target_name) - elif export_fact.local_name is not None: - origin = local_aliases_by_file.get(file_path, {}).get(export_fact.local_name) - if origin is None and (file_path, export_fact.local_name) in symbol_nodes: - origin = (file_path, export_fact.local_name) - if origin is None: - continue - named_exports_by_file.setdefault(file_path, {})[export_fact.exported_name] = origin - if origin[0] != file_path: - source_id = source_file_id.get(file_path) - if source_id is not None: - add_edge( - source_id, - _make_id(str(path_by_resolved.get(origin[0], origin[0]))), - "re_exports", - "export", - export_fact.line, - export_fact.file_path, - ) - - def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tuple[Path, str]] | None = None) -> tuple[Path, str]: - target_path = target_path.resolve() - key = (target_path, imported_name) - if seen is None: - seen = set() - if key in seen: - return key - seen.add(key) - origin = named_exports_by_file.get(target_path, {}).get(imported_name) - if origin is not None: - return resolve_exported_origin(origin[0], origin[1], seen) - for star_target in star_exports_by_file.get(target_path, []): - star_key = (star_target, imported_name) - if star_key in symbol_nodes: - return star_key - resolved = resolve_exported_origin(star_target, imported_name, seen) - if resolved in symbol_nodes: - return resolved - return key - - for import_fact in facts.imports: - source_id = source_file_id.get(import_fact.file_path.resolve()) - if source_id is None: - continue - origin_path, origin_symbol = resolve_exported_origin( - import_fact.target_path, - import_fact.imported_name, - ) - target_id = symbol_nodes.get((origin_path, origin_symbol)) - if target_id is None: - continue - add_edge( - source_id, - target_id, - "imports", - "import", - import_fact.line, - import_fact.file_path, - ) - - # #1146: emit file-to-file imports_from edges for package-form submodule imports. - for from_path, to_path, line in facts.module_imports: - try: - from_rel = from_path.relative_to(root) - to_rel = to_path.relative_to(root) - except ValueError: - continue - source_id = _make_id(_file_stem(from_rel)) - target_id = _make_id(_file_stem(to_rel)) - add_edge(source_id, target_id, "imports_from", "submodule_import", line, from_path) - - for use_fact in facts.uses: - file_path = use_fact.file_path.resolve() - target_id = None - unresolved_origin = local_aliases_by_file.get(file_path, {}).get(use_fact.local_name) - if unresolved_origin is not None: - origin_path, origin_symbol = resolve_exported_origin(*unresolved_origin) - target_id = symbol_nodes.get((origin_path, origin_symbol)) - if target_id is None and use_fact.relation in ("inherits", "implements"): - # Same-file fallback for HERITAGE only: a base declared in the same - # file (`class X extends Y`, `interface A extends B`) has no import - # alias, so resolve it directly against the file's own symbol nodes. - # Scoped to heritage because same-file calls/uses already resolve via - # the dedicated call-graph pass; widening this would duplicate those - # edges. Import resolution still takes precedence (#1095). - target_id = symbol_nodes.get((file_path, use_fact.local_name)) - if target_id is None: - continue - add_edge( - use_fact.source_id, - target_id, - use_fact.relation, - use_fact.context, - use_fact.line, - use_fact.file_path, - ) - - -def _parse_js_tree(path: Path): - try: - from tree_sitter import Language, Parser - # .vue embeds the script in non-JS markup; mask it out and parse the - # close tag + pos = m.end() + if lang is None: + lang_m = _VUE_SCRIPT_LANG_RE.search(m.group(1)) + if lang_m: + lang = lang_m.group(1).lower() + out.append(_blank(src[pos:])) + return "".join(out), lang + +def _source_key(source_file: str, root: Path) -> str: + if not source_file: + return "" + source_path = Path(source_file) + try: + return str(source_path.resolve().relative_to(root)) + except Exception: + return str(source_path) + +def _node_disambiguation_source_key(node: dict, root: Path) -> str: + source_file = str(node.get("source_file", "")) + if source_file: + return _source_key(source_file, root) + return _source_key(str(node.get("origin_file", "")), root) + +def _disambiguate_colliding_node_ids( + nodes: list[dict], + edges: list[dict], + raw_calls: list[dict], + root: Path, +) -> None: + """Rewrite only colliding node IDs, using source path as the disambiguator. + + Module anchor nodes (#1327) are exempt: ``import CoreKit`` from three files + yields three ``type=module`` nodes with the same id but different + source_files. Those are the *same* module, not distinct same-named symbols, + so they must collapse to one shared node — disambiguating them by path would + scatter a single module across N file-qualified duplicates. + """ + by_id: dict[str, list[dict]] = {} + for node in nodes: + if node.get("type") in ("module", "namespace"): + continue + nid = node.get("id") + if isinstance(nid, str) and nid: + by_id.setdefault(nid, []).append(node) + + remap: dict[tuple[str, str], str] = {} + ambiguous_ids: set[str] = set() + for old_id, group in by_id.items(): + source_keys = {_node_disambiguation_source_key(node, root) for node in group} + if len(group) < 2 or len(source_keys) < 2: + continue + ambiguous_ids.add(old_id) + # Salt the colliding id with the *path* it came from. The naive salt is + # ``_make_id(source_key, old_id)`` — source_key is the raw repo-relative + # path. But _make_id collapses every separator, so two DISTINCT paths + # whose only difference is a separator-vs-inner-punctuation swap + # (``a/b/c.md`` vs ``a.b/c.md``, ``foo/bar_baz.md`` vs ``foo_bar/baz.md``) + # normalize to the SAME salted id and still collide (#1522 — the residual + # of #1504 the 0.9.0 full-path stem didn't reach). When that happens, + # append a short stable hash of the *raw* source_key, which IS injective + # over distinct paths, so the colliders separate. Computed in code from + # source_file (never trusted from the LLM), so AST↔semantic parity holds. + naive: dict[str, str] = {} # source_key -> _make_id(source_key, old_id) + for source_key in source_keys: + if source_key: + naive[source_key] = _make_id(source_key, old_id) + # source_keys that, after normalization, are not unique among themselves. + seen: dict[str, int] = {} + for nid in naive.values(): + seen[nid] = seen.get(nid, 0) + 1 + needs_hash = {sk for sk, nid in naive.items() if seen.get(nid, 0) > 1} + for node in group: + source_key = _node_disambiguation_source_key(node, root) + if not source_key: + continue + if source_key in needs_hash: + salt = hashlib.sha1(source_key.encode("utf-8")).hexdigest()[:6] + new_id = _make_id(source_key, old_id, salt) + else: + new_id = naive.get(source_key) or _make_id(source_key, old_id) + remap[(old_id, source_key)] = new_id + if new_id != old_id: + node["id"] = new_id + + if not remap: + # No colliding ids to salt apart, but the transient `target_file` hint an + # importer stamps on every resolved import (#1814) still has to be dropped + # here — this early exit skips the edge loop below, so without it a + # non-colliding import would carry its absolute path into graph.json. + for edge in edges: + edge.pop("target_file", None) + return + + unambiguous_remaps: dict[str, str] = {} + for old_id, group in by_id.items(): + if old_id in ambiguous_ids: + continue + candidates = { + node["id"] for node in group + if isinstance(node.get("id"), str) and node["id"] != old_id + } + if len(candidates) == 1: + unambiguous_remaps[old_id] = next(iter(candidates)) + + # A C/ObjC/C++ `#include "foo.h"` / `#import "foo.h"` resolves to the header's + # file node, but `foo.h` and its sibling `foo.c`/`foo.m`/`foo.cpp` collapse to + # the same `foo` file id, so disambiguation salts them apart by path. A + # cross-file import edge from a THIRD file carries neither salt's source_key, so + # the (target, edge_source_key) lookup misses and the edge dangles on the now + # dead `foo` id. Repoint those import edges to the HEADER variant (the include + # always targeted the header), keyed by the original colliding id (#1475). + _HEADER_SUFFIXES = (".h", ".hpp", ".hh", ".hxx") + header_remaps: dict[str, str] = {} + for old_id in ambiguous_ids: + for node in by_id.get(old_id, []): + sk = _node_disambiguation_source_key(node, root) + if sk and Path(sk).suffix.lower() in _HEADER_SUFFIXES: + new_id = remap.get((old_id, sk)) + if new_id: + header_remaps[old_id] = new_id + break + + for edge in edges: + edge_source_key = _source_key(str(edge.get("source_file", "")), root) + source_key = (edge.get("source", ""), edge_source_key) + # An import/re-export edge's target is a FILE node that can collapse with a + # same-basename cross-extension sibling (foo.ts vs foo.mjs, #1814). Keying + # its target salt by the IMPORTER's own source_file mis-points it back at the + # importer's variant (a self-loop). When the emitter stamped the resolved + # target file, key the target salt by THAT file so the salt lands on the + # correct sibling. Generalizes the #1475 C/ObjC header carve-out (below) to + # every language and to re_exports. `pop` it as we consume it: this is the + # hint's only reader, and its absolute path must not persist into graph.json. + target_file = edge.pop("target_file", None) + if target_file and edge.get("relation") in ("imports", "imports_from", "re_exports"): + target_edge_key = _source_key(str(target_file), root) + else: + target_edge_key = edge_source_key + target_key = (edge.get("target", ""), target_edge_key) + if source_key in remap: + edge["source"] = remap[source_key] + elif edge.get("source") in unambiguous_remaps: + edge["source"] = unambiguous_remaps[str(edge["source"])] + # imports/imports_from always target a header file, so they must resolve to + # the header variant BEFORE the same-source-file salt is considered. Keying + # the import target by the importer's own source file mis-points a `.m` + # importing its own `.h` back at itself (self-loop), and is wrong for any + # cross-file import whose importer shares the colliding id (#1475). + if (edge.get("relation") in ("imports", "imports_from") + and edge.get("target") in header_remaps): + edge["target"] = header_remaps[str(edge["target"])] + elif target_key in remap: + edge["target"] = remap[target_key] + elif edge.get("target") in unambiguous_remaps: + edge["target"] = unambiguous_remaps[str(edge["target"])] + + for raw_call in raw_calls: + call_source_key = _source_key(str(raw_call.get("source_file", "")), root) + caller_key = (raw_call.get("caller_nid", ""), call_source_key) + if caller_key in remap: + raw_call["caller_nid"] = remap[caller_key] + elif raw_call.get("caller_nid") in unambiguous_remaps: + raw_call["caller_nid"] = unambiguous_remaps[str(raw_call["caller_nid"])] + +def _is_type_like_definition(node: dict) -> bool: + if node.get("type") == "namespace": + return False + label = str(node.get("label", "")).strip() + if not label: + return False + if label.endswith(")") or label.startswith("."): + return False + if "." in label: + return False + return node.get("file_type") == "code" + +def _js_source_path(source_file: str, root: Path) -> Path | None: + if not source_file: + return None + path = Path(source_file) + if not path.is_absolute(): + path = root / path + try: + return path.resolve() + except Exception: + return path + +def _apply_symbol_resolution_facts( + paths: list[Path], + nodes: list[dict], + edges: list[dict], + root: Path, + facts: _SymbolResolutionFacts, +) -> None: + """Apply language-provided import/export/use facts to graph edges.""" + if not ( + facts.declarations + or facts.imports + or facts.aliases + or facts.exports + or facts.star_exports + or facts.namespace_exports + or facts.uses + or facts.module_imports + ): + return + + path_by_resolved = {path.resolve(): path for path in paths} + source_file_id = {path.resolve(): _make_id(str(path)) for path in paths} + symbol_nodes: dict[tuple[Path, str], str] = {} + for node in nodes: + source_path = _js_source_path(str(node.get("source_file", "")), root) + if source_path is None: + continue + label = str(node.get("label", "")).strip().strip("()").lstrip(".") + if label and node.get("id"): + symbol_nodes[(source_path, label)] = str(node["id"]) + + def ensure_symbol_node(path: Path, name: str, line: int) -> str: + resolved_path = path.resolve() + existing = symbol_nodes.get((resolved_path, name)) + if existing is not None: + return existing + node_id = _make_id(_file_stem(path), name) + symbol_nodes[(resolved_path, name)] = node_id + nodes.append({ + "id": node_id, + "label": name, + "file_type": "code", + "source_file": str(path), + "source_location": f"L{line}", + }) + return node_id + + existing_edges = { + ( + str(edge.get("source")), + str(edge.get("target")), + str(edge.get("relation")), + str(edge.get("context") or ""), + ) + for edge in edges + } + + def add_edge(source: str, target: str, relation: str, context: str, line: int, source_path: Path, target_file: str | None = None, local_alias: str | None = None) -> None: + key = (source, target, relation, context or "") + if key in existing_edges: + return + existing_edges.add(key) + edge = { + "source": source, + "target": target, + "relation": relation, + "context": context, + "confidence": "EXTRACTED", + "source_file": str(source_path), + "source_location": f"L{line}", + "weight": 1.0, + } + # A re-export edge's target is a FILE node that can collapse with a + # same-basename cross-extension sibling; stamp the resolved target file so + # the id-disambiguation salt is keyed by the TARGET, not the importer (#1814). + if target_file is not None: + edge["target_file"] = target_file + # The local name this import bound in the importing file, when it differs + # from the target's own name (`from pkg import mod as alias`) -- lets the + # cross-file member-call resolver match `alias.func()` (#2082). + if local_alias is not None: + edge["local_alias"] = local_alias + edges.append(edge) + + for declaration in facts.declarations: + ensure_symbol_node(declaration.file_path, declaration.name, declaration.line) + + local_aliases_by_file: dict[Path, dict[str, tuple[Path, str]]] = {} + for import_fact in facts.imports: + file_path = import_fact.file_path.resolve() + local_aliases_by_file.setdefault(file_path, {})[import_fact.local_name] = ( + import_fact.target_path.resolve(), + import_fact.imported_name, + ) + + pending_aliases_by_file: dict[Path, list[_SymbolAliasFact]] = {} + for alias_fact in facts.aliases: + pending_aliases_by_file.setdefault(alias_fact.file_path.resolve(), []).append(alias_fact) + + for file_path, aliases in pending_aliases_by_file.items(): + local_aliases = local_aliases_by_file.setdefault(file_path, {}) + changed = True + while changed: + changed = False + for alias_fact in aliases: + if alias_fact.alias in local_aliases: + continue + origin = local_aliases.get(alias_fact.target_name) + if origin is not None: + local_aliases[alias_fact.alias] = origin + changed = True + + named_exports_by_file: dict[Path, dict[str, tuple[Path, str]]] = {} + star_exports_by_file: dict[Path, list[Path]] = {} + + for star_fact in facts.star_exports: + source_path = star_fact.file_path.resolve() + target_path = star_fact.target_path.resolve() + star_exports_by_file.setdefault(source_path, []).append(target_path) + source_id = source_file_id.get(source_path) + if source_id is not None: + add_edge( + source_id, + _make_id(str(path_by_resolved.get(target_path, target_path))), + "re_exports", + "export", + star_fact.line, + star_fact.file_path, + target_file=str(path_by_resolved.get(target_path, target_path)), + ) + + for namespace_fact in facts.namespace_exports: + source_path = namespace_fact.file_path.resolve() + target_path = namespace_fact.target_path.resolve() + namespace_id = ensure_symbol_node( + namespace_fact.file_path, + namespace_fact.exported_name, + namespace_fact.line, + ) + named_exports_by_file.setdefault(source_path, {})[ + namespace_fact.exported_name + ] = (source_path, namespace_fact.exported_name) + source_id = source_file_id.get(source_path) + if source_id is not None: + add_edge( + source_id, + namespace_id, + "contains", + "namespace_export", + namespace_fact.line, + namespace_fact.file_path, + ) + add_edge( + source_id, + _make_id(str(path_by_resolved.get(target_path, target_path))), + "re_exports", + "export", + namespace_fact.line, + namespace_fact.file_path, + target_file=str(path_by_resolved.get(target_path, target_path)), + ) + + for export_fact in facts.exports: + file_path = export_fact.file_path.resolve() + origin: tuple[Path, str] | None = None + if export_fact.target_path is not None and export_fact.target_name is not None: + origin = (export_fact.target_path.resolve(), export_fact.target_name) + elif export_fact.local_name is not None: + origin = local_aliases_by_file.get(file_path, {}).get(export_fact.local_name) + if origin is None and (file_path, export_fact.local_name) in symbol_nodes: + origin = (file_path, export_fact.local_name) + if origin is None: + continue + named_exports_by_file.setdefault(file_path, {})[export_fact.exported_name] = origin + if origin[0] != file_path: + source_id = source_file_id.get(file_path) + if source_id is not None: + add_edge( + source_id, + _make_id(str(path_by_resolved.get(origin[0], origin[0]))), + "re_exports", + "export", + export_fact.line, + export_fact.file_path, + target_file=str(path_by_resolved.get(origin[0], origin[0])), + ) + + def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tuple[Path, str]] | None = None) -> tuple[Path, str]: + target_path = target_path.resolve() + key = (target_path, imported_name) + if seen is None: + seen = set() + if key in seen: + return key + seen.add(key) + origin = named_exports_by_file.get(target_path, {}).get(imported_name) + if origin is not None: + return resolve_exported_origin(origin[0], origin[1], seen) + for star_target in star_exports_by_file.get(target_path, []): + star_key = (star_target, imported_name) + if star_key in symbol_nodes: + return star_key + resolved = resolve_exported_origin(star_target, imported_name, seen) + if resolved in symbol_nodes: + return resolved + return key + + for import_fact in facts.imports: + source_id = source_file_id.get(import_fact.file_path.resolve()) + if source_id is None: + continue + origin_path, origin_symbol = resolve_exported_origin( + import_fact.target_path, + import_fact.imported_name, + ) + target_id = symbol_nodes.get((origin_path, origin_symbol)) + if target_id is None: + continue + add_edge( + source_id, + target_id, + "imports", + "import", + import_fact.line, + import_fact.file_path, + ) + + # #1146: emit file-to-file imports_from edges for package-form submodule imports. + for from_path, to_path, line, local_name in facts.module_imports: + try: + from_rel = from_path.relative_to(root) + to_rel = to_path.relative_to(root) + except ValueError: + continue + source_id = _make_id(_file_stem(from_rel)) + target_id = _make_id(_file_stem(to_rel)) + add_edge( + source_id, target_id, "imports_from", "submodule_import", line, from_path, + local_alias=local_name if local_name != to_path.stem else None, + ) + + for use_fact in facts.uses: + file_path = use_fact.file_path.resolve() + target_id = None + unresolved_origin = local_aliases_by_file.get(file_path, {}).get(use_fact.local_name) + if unresolved_origin is not None: + origin_path, origin_symbol = resolve_exported_origin(*unresolved_origin) + target_id = symbol_nodes.get((origin_path, origin_symbol)) + if target_id is None and use_fact.relation in ("inherits", "implements"): + # Same-file fallback for HERITAGE only: a base declared in the same + # file (`class X extends Y`, `interface A extends B`) has no import + # alias, so resolve it directly against the file's own symbol nodes. + # Scoped to heritage because same-file calls/uses already resolve via + # the dedicated call-graph pass; widening this would duplicate those + # edges. Import resolution still takes precedence (#1095). + target_id = symbol_nodes.get((file_path, use_fact.local_name)) + if target_id is None: + continue + add_edge( + use_fact.source_id, + target_id, + use_fact.relation, + use_fact.context, + use_fact.line, + use_fact.file_path, + ) + +def _parse_js_tree(path: Path): + try: + from tree_sitter import Language, Parser + # .vue embeds the script in non-JS markup; mask it out and parse the + # +

{projects.length}

+""", + ) + _write(root / "src/lib/content.ts", "export const projects = [1];\n") + + result = extract( + [root / "src/routes/page.svelte", root / "src/lib/content.ts"], + cache_root=root, + ) + + _assert_no_root_slug(result, root) + + content_file_nodes = [ + n for n in result["nodes"] if n["id"] == "src_lib_content" + ] + assert len(content_file_nodes) == 1 + assert content_file_nodes[0]["source_file"] == "src/lib/content.ts" + + page_id = _file_node_id(Path("src/routes/page.svelte")) + edge_pairs = { + (e["source"], e["target"], e["relation"]) for e in result["edges"] + } + assert (page_id, "src_lib_content", "imports_from") in edge_pairs + assert (page_id, "src_lib_content", "dynamic_import") in edge_pairs + + +def test_astro_unresolved_relative_import_id_still_portable(tmp_path): + """A rescued import whose target does NOT exist still mints a stub (so the + edge survives as a hint), but the stub id must be the repo-relative form, + not the absolute-path slug — the belt-and-braces remap in the + relativization pass (#2195).""" + root = Path(os.path.realpath(tmp_path)) + _write( + root / "src/pages/index.astro", + """--- +import { gone } from '../missing/nowhere'; +--- +

x

+""", + ) + result = extract([root / "src/pages/index.astro"], cache_root=root) + + _assert_no_root_slug(result, root) + stub = [n for n in result["nodes"] if n.get("label") == "../missing/nowhere"] + assert len(stub) == 1 + assert stub[0]["id"] == "src_missing_nowhere" diff --git a/tests/test_atomic_writes.py b/tests/test_atomic_writes.py new file mode 100644 index 000000000..968cb0b96 --- /dev/null +++ b/tests/test_atomic_writes.py @@ -0,0 +1,134 @@ +"""Tests for atomic JSON writes (graph.json / manifest.json). + +A crash, kill, or disk-full mid-write must not leave a truncated/corrupt file +that a later load chokes on. `write_text_atomic` writes a temp file in the same +directory then `os.replace`s it into place; on failure the original is untouched. +""" +import json +import os + +import pytest + +from graphify.paths import write_text_atomic + + +def test_write_text_atomic_writes_and_leaves_no_tmp(tmp_path): + p = tmp_path / "out" / "graph.json" # parent doesn't exist yet + write_text_atomic(p, '{"a": 1}') + assert json.loads(p.read_text()) == {"a": 1} + # No leftover temp file in the target directory. + assert [x.name for x in p.parent.iterdir()] == ["graph.json"] + + +def test_write_text_atomic_preserves_existing_on_failure(tmp_path, monkeypatch): + p = tmp_path / "graph.json" + p.write_text("original", encoding="utf-8") + + def boom(src, dst): + raise OSError("simulated disk full") + + monkeypatch.setattr(os, "replace", boom) + with pytest.raises(OSError): + write_text_atomic(p, "content-that-must-not-land") + + # The original file is intact and the temp file was cleaned up. + assert p.read_text() == "original" + assert sorted(x.name for x in tmp_path.iterdir()) == ["graph.json"] + + +def test_write_text_atomic_preserves_existing_mode(tmp_path): + # An atomic replace must not tighten a 0644 file to mkstemp's 0600 default. + p = tmp_path / "graph.json" + p.write_text("{}", encoding="utf-8") + os.chmod(p, 0o644) + write_text_atomic(p, '{"x": 1}') + assert (os.stat(p).st_mode & 0o777) == 0o644 + + +def test_write_text_atomic_new_file_respects_umask(tmp_path): + # A brand-new file must land at the umask default (e.g. 0644), NOT mkstemp's + # 0600 — otherwise every fresh graph.json would be owner-only. + p = tmp_path / "new.json" + write_text_atomic(p, "{}") + umask = os.umask(0) + os.umask(umask) + assert (os.stat(p).st_mode & 0o777) == (0o666 & ~umask) + + +def test_write_text_atomic_writes_through_symlink(tmp_path): + # Shared-output setups symlink graph.json to shared storage; the atomic write + # must update the target and keep the link, not replace it with a real file. + target = tmp_path / "real.json" + target.write_text("old", encoding="utf-8") + link = tmp_path / "link.json" + link.symlink_to(target) + write_text_atomic(link, "new") + assert link.is_symlink() + assert target.read_text() == "new" + + +def test_write_json_atomic_roundtrip(tmp_path): + from graphify.paths import write_json_atomic + + p = tmp_path / "g.json" + write_json_atomic(p, {"nodes": [1, 2], "x": "é"}, indent=2) + assert json.loads(p.read_text()) == {"nodes": [1, 2], "x": "é"} + assert not any(name.name.endswith(".tmp") for name in tmp_path.iterdir()) + + +def test_to_json_writes_atomically_no_tmp_leftover(tmp_path): + import networkx as nx + from graphify.export import to_json + + G = nx.Graph() + G.add_node("a", label="a", file_type="code") + G.add_node("b", label="b", file_type="code") + G.add_edge("a", "b") + out = tmp_path / "graph.json" + assert to_json(G, {}, str(out), force=True) is True + json.loads(out.read_text()) # valid JSON + assert not any(x.name.endswith(".tmp") for x in tmp_path.iterdir()) + + +def test_save_manifest_writes_atomically(tmp_path): + from graphify.detect import save_manifest + + (tmp_path / "a.py").write_text("x = 1\n", encoding="utf-8") + mpath = tmp_path / "graphify-out" / "manifest.json" + save_manifest({"code": [str(tmp_path / "a.py")]}, manifest_path=str(mpath), + kind="both", root=tmp_path) + assert json.loads(mpath.read_text()) # non-empty, valid JSON + assert not any(x.name.endswith(".tmp") for x in mpath.parent.iterdir()) + + +def test_write_text_atomic_windows_permission_fallback(tmp_path, monkeypatch): + """On Windows os.replace raises PermissionError when the destination is + briefly locked (antivirus, an open reader); the copy-then-delete fallback + must still land the new content and leave no temp file.""" + p = tmp_path / "graph.json" + p.write_text("original", encoding="utf-8") + + real_replace = os.replace + calls = {"n": 0} + + def flaky_replace(src, dst): + calls["n"] += 1 + raise PermissionError("simulated WinError 5") + + monkeypatch.setattr(os, "replace", flaky_replace) + write_text_atomic(p, "new-content") + + assert calls["n"] == 1 # the fallback path was actually exercised + assert p.read_text() == "new-content" + assert sorted(x.name for x in tmp_path.iterdir()) == ["graph.json"] + + +def test_write_json_atomic_ensure_ascii_false_preserves_utf8(tmp_path): + from graphify.paths import write_json_atomic + + p = tmp_path / "g.json" + write_json_atomic(p, {"label": "Wörker 数据"}, ensure_ascii=False) + raw = p.read_text(encoding="utf-8") + assert "Wörker 数据" in raw # raw UTF-8, not \\uXXXX escapes + assert "\\u" not in raw + assert json.loads(raw) == {"label": "Wörker 数据"} diff --git a/tests/test_benchmark_raw_graph.py b/tests/test_benchmark_raw_graph.py new file mode 100644 index 000000000..51da5fe02 --- /dev/null +++ b/tests/test_benchmark_raw_graph.py @@ -0,0 +1,103 @@ +"""#2212: run_benchmark must accept a raw --no-cluster graph.json. + +The clustered writer stores edges under "links" (networkx node-link default); +the raw --no-cluster writer stores them under "edges". Consumers calling +node_link_graph(data, edges="links") raised KeyError: 'links' on the raw +shape — the `except TypeError` fallback only covered old networkx versions, +not the missing key. +""" +from __future__ import annotations +import json + +from graphify.benchmark import run_benchmark + + +def _graph_payload(edges_key: str) -> dict: + # Raw extract shape: top-level nodes/edges/hyperedges + token counters. + nodes = [ + { + "id": "auth_flow", + "label": "authentication flow", + "source_file": "auth.py", + "source_location": "L1", + }, + { + "id": "login_handler", + "label": "user login authentication handler", + "source_file": "auth.py", + "source_location": "L10", + }, + { + "id": "main_entry", + "label": "main entry point", + "source_file": "main.py", + "source_location": "L1", + }, + ] + edges = [ + { + "id": "edge_1", + "source": "auth_flow", + "target": "login_handler", + "relation": "calls", + "confidence": "EXTRACTED", + "confidence_score": 1.0, + }, + { + "id": "edge_2", + "source": "login_handler", + "target": "main_entry", + "relation": "used_by", + "confidence": "EXTRACTED", + "confidence_score": 1.0, + }, + ] + return { + "nodes": nodes, + edges_key: edges, + "hyperedges": [], + "input_tokens": 0, + "output_tokens": 0, + } + + +def test_run_benchmark_raw_edges_keyed_graph(tmp_path): + """A --no-cluster graph.json ("edges" key) must not raise KeyError.""" + graph_file = tmp_path / "graph.json" + graph_file.write_text(json.dumps(_graph_payload("edges")), encoding="utf-8") + + result = run_benchmark(graph_path=str(graph_file), corpus_words=5_000) + + assert "error" not in result + assert result["nodes"] == 3 + assert result["edges"] == 2 + assert result["reduction_ratio"] > 0 + assert any( + "authentication" in p["question"] for p in result["per_question"] + ) + + +def test_run_benchmark_links_keyed_graph(tmp_path): + """The clustered writer's "links" key keeps working identically.""" + graph_file = tmp_path / "graph.json" + graph_file.write_text(json.dumps(_graph_payload("links")), encoding="utf-8") + + result = run_benchmark(graph_path=str(graph_file), corpus_words=5_000) + + assert "error" not in result + assert result["nodes"] == 3 + assert result["edges"] == 2 + assert result["reduction_ratio"] > 0 + + +def test_raw_and_links_graphs_benchmark_identically(tmp_path): + """Both spellings of the same graph must produce the same stats.""" + raw_file = tmp_path / "raw.json" + raw_file.write_text(json.dumps(_graph_payload("edges")), encoding="utf-8") + links_file = tmp_path / "links.json" + links_file.write_text(json.dumps(_graph_payload("links")), encoding="utf-8") + + raw = run_benchmark(graph_path=str(raw_file), corpus_words=5_000) + links = run_benchmark(graph_path=str(links_file), corpus_words=5_000) + + assert raw == links diff --git a/tests/test_build.py b/tests/test_build.py index 2d8bfdd60..0851ba75e 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -58,6 +58,48 @@ def test_build_from_json_edge_count(): G = build_from_json(load_extraction()) assert G.number_of_edges() == 4 +def test_null_weight_edge_builds_and_clusters(tmp_path): + """#1960: an explicit ``"weight": null`` (JSON null -> None) used to survive + ``.get("weight", 1.0)`` and crash Louvain/Leiden modularity with a TypeError. + It must now coerce to the 1.0 default, build, and cluster without raising.""" + from graphify.cluster import cluster + extraction = { + "nodes": [ + {"id": "a", "label": "A", "file_type": "code", "source_file": "a.py"}, + {"id": "b", "label": "B", "file_type": "code", "source_file": "b.py"}, + {"id": "c", "label": "C", "file_type": "code", "source_file": "c.py"}, + ], + "edges": [ + {"source": "a", "target": "b", "relation": "references", "weight": None, + "confidence_score": None}, + {"source": "b", "target": "c", "relation": "references", "weight": 2.5}, + ], + } + G = build_from_json(extraction) + assert G["a"]["b"]["weight"] == 1.0 # null coerced to default + assert G["a"]["b"]["confidence_score"] == 1.0 # null confidence_score too + assert G["b"]["c"]["weight"] == 2.5 # a valid weight is preserved + cluster(G) # must not raise (Louvain/Leiden modularity) + + +def test_malformed_weights_normalize(): + """Non-numeric / NaN / inf / negative weights fall back to 1.0 (the backends + reject them); a missing weight key is left absent (backends default it).""" + extraction = { + "nodes": [{"id": f"n{i}", "label": str(i), "file_type": "code", + "source_file": f"{i}.py"} for i in range(4)], + "edges": [ + {"source": "n0", "target": "n1", "relation": "references", "weight": "3.5"}, + {"source": "n1", "target": "n2", "relation": "references", "weight": float("nan")}, + {"source": "n2", "target": "n3", "relation": "references", "weight": -4}, + ], + } + G = build_from_json(extraction) + assert G["n0"]["n1"]["weight"] == 3.5 # numeric string coerces + assert G["n1"]["n2"]["weight"] == 1.0 # NaN -> default + assert G["n2"]["n3"]["weight"] == 1.0 # negative -> default + + def test_nodes_have_label(): G = build_from_json(load_extraction()) assert G.nodes["n_transformer"]["label"] == "Transformer" @@ -93,6 +135,145 @@ def test_legacy_edge_from_to_canonicalized(): assert G.number_of_edges() == 1 +def test_legacy_node_name_path_aliases_folded(): + """#2194: nodes carrying `name`/`path` instead of `label`/`source_file` must + be canonicalized before validation, not enter the graph as label-less + ghosts. After build the canonicalized dict also passes validation.""" + from graphify.validate import validate_extraction + ext = {"nodes": [{"id": "n1", "name": "Foo", "path": "a/b.md", "file_type": "concept"}], + "edges": [], "input_tokens": 0, "output_tokens": 0} + G = build_from_json(ext) + attrs = G.nodes["n1"] + assert attrs["label"] == "Foo" + assert attrs["source_file"] == "a/b.md" + assert "name" not in attrs + assert "path" not in attrs + # build_from_json canonicalizes in place; the extraction dict must now be + # schema-valid (no missing-field errors for the alias node). + assert not [e for e in validate_extraction(ext) if "missing required field" in e] + + +def test_legacy_edge_type_confidence_score_aliases_folded(): + """#2194: edges carrying `type`/`confidence_score` instead of + `relation`/`confidence` fold to canonical fields. Recovery confidence is + INFERRED (never EXTRACTED — alias recovery is not provenance) and the + companion confidence_score float is retained, not popped.""" + ext = {"nodes": [{"id": "n1", "label": "A", "file_type": "code", "source_file": "a.py"}, + {"id": "n2", "label": "B", "file_type": "code", "source_file": "b.py"}], + "edges": [{"source": "n1", "target": "n2", "type": "references", + "confidence_score": 0.9, "source_file": "a.py"}], + "input_tokens": 0, "output_tokens": 0} + G = build_from_json(ext) + data = edge_data(G, "n1", "n2") + assert data["relation"] == "references" + assert data["confidence"] == "INFERRED" + assert data["confidence_score"] == 0.9 + assert "type" not in data + + +def test_node_alias_canonical_field_wins(): + """#2194: when both the canonical field and its alias are present, the + canonical value wins and the alias key is left untouched.""" + ext = {"nodes": [{"id": "n1", "label": "Real", "name": "Alias", + "file_type": "code", "source_file": "a.py"}], + "edges": [], "input_tokens": 0, "output_tokens": 0} + G = build_from_json(ext) + assert G.nodes["n1"]["label"] == "Real" + assert G.nodes["n1"]["name"] == "Alias" # preserved, not consumed + + +def test_alias_node_ghost_merges_into_ast_twin(): + """#2194: an alias-only semantic node (name/path) must participate in the + AST/LLM ghost merge once folded — same label and file as an AST node with a + different id collapses into the AST node instead of surviving as a ghost.""" + ext = {"nodes": [ + {"id": "src_foo_helper", "label": "helper", "file_type": "code", + "source_file": "src/foo.py", "_origin": "ast", "source_location": "L10"}, + {"id": "helper_ghost", "name": "helper", "path": "src/foo.py", + "file_type": "code"}, + ], "edges": [], "input_tokens": 0, "output_tokens": 0} + G = build_from_json(ext) + assert "src_foo_helper" in G.nodes + assert "helper_ghost" not in G.nodes + assert G.number_of_nodes() == 1 + + +def test_alias_node_gets_nonempty_norm_label(tmp_path): + """#2194: a recovered alias node must serialize with a non-empty norm_label + so query/explain can find it.""" + from graphify.export import to_json + ext = {"nodes": [{"id": "n1", "name": "Foo", "path": "a/b.md", "file_type": "concept"}], + "edges": [], "input_tokens": 0, "output_tokens": 0} + G = build_from_json(ext) + out = tmp_path / "graph.json" + assert to_json(G, {}, str(out)) + data = json.loads(out.read_text()) + node = next(n for n in data["nodes"] if n["id"] == "n1") + assert node["norm_label"] == "foo" + + +def test_extraction_warning_breakdown_by_cause(capsys): + """#2194: a mixed batch of schema errors must report per-cause counts, not + just the first error.""" + ext = {"nodes": [ + {"id": "n1", "label": "A", "file_type": "code", "source_file": "a.py"}, + {"id": "n2", "label": "B", "file_type": "code", "source_file": "b.py"}, + # two nodes missing label (and carrying no name alias) + {"id": "x1", "file_type": "code", "source_file": "x.py"}, + {"id": "x2", "file_type": "code", "source_file": "x.py"}, + ], "edges": [ + # three edges missing relation (and carrying no type alias) + {"source": "n1", "target": "n2", "confidence": "EXTRACTED", "source_file": "a.py"}, + {"source": "n2", "target": "n1", "confidence": "EXTRACTED", "source_file": "a.py"}, + {"source": "n1", "target": "x1", "confidence": "EXTRACTED", "source_file": "a.py"}, + ], "input_tokens": 0, "output_tokens": 0} + build_from_json(ext) + err = capsys.readouterr().err + assert "2x missing required field 'label'" in err + assert "3x missing required field 'relation'" in err + + +def test_absolute_derived_semantic_ids_rekeyed(tmp_path): + """#2197: a semantic fragment whose ids were derived from an ABSOLUTE + source_file (Windows detect() emits them) must re-key to the canonical + repo-relative stem instead of ghosting against the existing graph.""" + from graphify.ids import make_id + (tmp_path / "docs").mkdir() + abs_sf = str(tmp_path / "docs" / "DATAFLOW.md") + abs_stem = make_id(str(tmp_path / "docs" / "DATAFLOW")) + ext = {"nodes": [ + {"id": abs_stem, "label": "DATAFLOW.md", "file_type": "document", + "source_file": abs_sf}, + {"id": f"{abs_stem}_pipeline", "label": "Pipeline", "file_type": "concept", + "source_file": abs_sf}, + ], "edges": [ + {"source": abs_stem, "target": f"{abs_stem}_pipeline", "relation": "describes", + "confidence": "INFERRED", "source_file": abs_sf, "weight": 1.0}, + ], "input_tokens": 0, "output_tokens": 0} + G = build_from_json(ext, root=tmp_path) + assert "docs_dataflow" in G.nodes + assert "docs_dataflow_pipeline" in G.nodes + assert abs_stem not in G.nodes + assert G.nodes["docs_dataflow"]["source_file"] == "docs/DATAFLOW.md" + assert G.has_edge("docs_dataflow", "docs_dataflow_pipeline") + + +def test_absolute_derived_semantic_ids_rekeyed_backslash(tmp_path): + """#2197 (separator variant): the same absolute-derived-id fragment with + backslash separators in source_file re-keys identically.""" + from graphify.ids import make_id + (tmp_path / "docs").mkdir() + abs_sf = str(tmp_path / "docs" / "DATAFLOW.md").replace("/", "\\") + abs_stem = make_id(str(tmp_path / "docs" / "DATAFLOW")) + ext = {"nodes": [ + {"id": f"{abs_stem}_pipeline", "label": "Pipeline", "file_type": "concept", + "source_file": abs_sf}, + ], "edges": [], "input_tokens": 0, "output_tokens": 0} + G = build_from_json(ext, root=tmp_path) + assert "docs_dataflow_pipeline" in G.nodes + assert G.nodes["docs_dataflow_pipeline"]["source_file"] == "docs/DATAFLOW.md" + + def test_source_file_backslash_normalized(): """Windows backslash paths and POSIX paths for the same file must produce one node.""" extraction = { @@ -228,10 +409,12 @@ def test_ghost_merge_unique_located_node_still_merges(): assert G.has_edge("caller", "ast_render") -def test_ghost_merge_skipped_on_basename_collision(): - """#1257: when two files with the same basename both define a symbol with the - same label, the (basename, label) key is ambiguous and the semantic ghost - must not be merged into an arbitrary one of them.""" +def test_ghost_merge_uses_source_file_not_basename(): + """#2068: the ghost-merge key is the full source_file, not the bare basename. + A ghost from src/a/index.ts merges into THAT file's AST node (a_render), never + the unrelated same-basename b_render in src/b/index.ts. (Pre-#2068 the + (basename, label) key made ('index.ts','render') ambiguous and skipped the + merge; the directory-aware key resolves it precisely.)""" ext = { "nodes": [ {"id": "a_render", "label": "render", "file_type": "code", @@ -248,13 +431,71 @@ def test_ghost_merge_skipped_on_basename_collision(): "input_tokens": 0, "output_tokens": 0, } G = build_from_json(ext) - # The ghost survives: merging it into either a_render or b_render would - # pick an arbitrary winner (set iteration order over node_set). - assert "ghost_render" in G.nodes() - assert G.number_of_nodes() == 4 - assert G.has_edge("caller", "ghost_render") - assert not G.has_edge("caller", "a_render") + # Ghost merges into its same-file twin; the edge re-points to a_render only. + assert "ghost_render" not in G.nodes() + assert G.has_edge("caller", "a_render") assert not G.has_edge("caller", "b_render") + # The unrelated same-basename node in another directory is untouched. + assert "b_render" in G.nodes() + + +def test_ghost_merge_not_across_directories_same_basename(): + """#2068: two unrelated non-AST nodes with the same basename+label in + DIFFERENT directories must NOT be merged onto one survivor (the bug: bare + basename collapsed docs/product_a/index.md and docs/product_b/index.md).""" + ext = { + "nodes": [ + {"id": "docs_a_index", "label": "Quickstart", "file_type": "document", + "source_file": "docs/product_a/index.md", "source_location": "L1"}, + {"id": "docs_b_index", "label": "Quickstart", "file_type": "document", + "source_file": "docs/product_b/index.md"}, + {"id": "docs_hub", "label": "Docs", "file_type": "concept", + "source_file": "docs/hub.md", "source_location": "L1"}, + ], + "edges": [{"source": "docs_hub", "target": "docs_b_index", "relation": "links_to", + "confidence": "INFERRED", "source_file": "docs/hub.md"}], + "input_tokens": 0, "output_tokens": 0, + } + G = build_from_json(ext, directed=False) + # Both docs survive; the edge stays on the file it was authored against. + assert "docs_a_index" in G.nodes() and "docs_b_index" in G.nodes() + assert G.has_edge("docs_hub", "docs_b_index") + assert not G.has_edge("docs_hub", "docs_a_index") + + +def test_ghost_merge_non_ast_different_files_both_survive(): + """#1753: two NON-AST (semantic) nodes sharing (basename, label) but from + DIFFERENT files are distinct concepts with no AST canonical twin. They must + not be merged into an arbitrary survivor (which flipped run-to-run with the + hash seed); both survive, mirroring the AST/AST guard (#1257).""" + ext = { + "nodes": [ + {"id": "dir_a_update_build_merge", "label": "build_merge() function", + "file_type": "concept", "source_file": "dir_a/update.md", "source_location": "L10"}, + {"id": "dir_b_update_build_merge", "label": "build_merge() function", + "file_type": "concept", "source_file": "dir_b/update.md", "source_location": "L12"}, + ], + "edges": [], + } + G = build_from_json(ext, directed=False) + assert sorted(G.nodes()) == ["dir_a_update_build_merge", "dir_b_update_build_merge"] + + +def test_ghost_merge_non_ast_same_file_still_merges(): + """A genuine duplicate — two non-AST nodes with the SAME source_file and + label — is a real ghost and still collapses to one node (deterministically), + so #1753's fix doesn't leave same-file LLM duplicates behind.""" + ext = { + "nodes": [ + {"id": "a_foo", "label": "Foo", "file_type": "concept", + "source_file": "x/doc.md", "source_location": "L1"}, + {"id": "b_foo", "label": "Foo", "file_type": "concept", + "source_file": "x/doc.md", "source_location": "L2"}, + ], + "edges": [], + } + G = build_from_json(ext, directed=False) + assert G.number_of_nodes() == 1 def test_build_merge_preserves_call_edge_direction(tmp_path): @@ -505,6 +746,99 @@ def test_build_relativizes_absolute_source_file(tmp_path): assert sf == "src/main.py" +def test_build_from_json_ambiguous_old_stem_alias_stays_dangling(tmp_path): + """The #1504 old-stem alias (e.g. "ping.h" -> bare "ping") is meant to let a + stale-id edge from an un-re-extracted fragment still find its own file after + a rekey. But the old-stem form drops the extension and most of the path, so + two unrelated real files easily collapse onto the same bare alias (a C header + and a PHP script both named "ping", in different directories). A dangling + edge produced by an unrelated third file's own unscoped fallback id (e.g. the + C/C++ extractor's last-resort target for an #include it couldn't resolve to + a real path) must not silently ride that alias onto an arbitrary one of them + — it should stay dangling and get dropped, same as any other unresolvable + edge, rather than wire two unrelated files/languages together by accident.""" + root = tmp_path / "repo" + root.mkdir() + extraction = { + "nodes": [ + # Ids given in their canonical (post-extract.py, extension-stripped) + # form, matching what a real graphify update run would already have + # produced before build_from_json assembles the final graph. + {"id": "dev_monitoring_ping", "label": "ping.h", "file_type": "code", + "source_file": "Dev/monitoring/ping.h"}, + {"id": "www_pages_api_ping", "label": "ping.php", "file_type": "code", + "source_file": "www/pages/api/ping.php"}, + {"id": "dev_poker_server", "label": "server.cpp", "file_type": "code", + "source_file": "Dev/poker/server.cpp"}, + ], + "edges": [ + # The unscoped, deliberately-unresolved fallback edge a C/C++ #include + # resolver leaves behind when it can't find the header on disk. + {"source": "dev_poker_server", "target": "ping", "relation": "imports", + "confidence": "EXTRACTED", "source_file": "Dev/poker/server.cpp"}, + ], + } + G = build_from_json(extraction, root=root) + assert not G.has_edge("dev_poker_server", "dev_monitoring_ping") + assert not G.has_edge("dev_poker_server", "www_pages_api_ping") + + +def test_build_from_json_ambiguous_alias_detected_despite_header_impl_salting(tmp_path): + """A same-directory .h/.cpp pair collides on their shared pre-extension id + and gets salted apart into ids like "tools_aolserver_utility_h_..." — no + longer a clean new_stem prefix. The ambiguity check must still recognize + the salted header as a legitimate claimant for the bare old-stem alias (by + label, not id shape), so a real collision with an unrelated same-named PHP + file is still caught instead of the header silently dropping out of the + race and leaving the PHP file as the lone "unambiguous" winner (this + reproduced against the real depot: Tools/aolserver/utility.h and .cpp, + salted apart, let wwwapi.masque.com/pages/utility.php win the bare + "utility" alias uncontested).""" + root = tmp_path / "repo" + root.mkdir() + extraction = { + "nodes": [ + {"id": "tools_aolserver_utility_h_tools_aolserver_utility", "label": "utility.h", + "file_type": "code", "source_file": "Tools/aolserver/utility.h"}, + {"id": "tools_aolserver_utility_cpp_tools_aolserver_utility", "label": "utility.cpp", + "file_type": "code", "source_file": "Tools/aolserver/utility.cpp"}, + {"id": "wwwapi_masque_com_pages_utility", "label": "utility.php", + "file_type": "code", "source_file": "wwwapi.masque.com/pages/utility.php"}, + {"id": "dev_poker_server", "label": "server.cpp", "file_type": "code", + "source_file": "Dev/poker/server.cpp"}, + ], + "edges": [ + {"source": "dev_poker_server", "target": "utility", "relation": "imports", + "confidence": "EXTRACTED", "source_file": "Dev/poker/server.cpp"}, + ], + } + G = build_from_json(extraction, root=root) + assert not G.has_edge("dev_poker_server", "wwwapi_masque_com_pages_utility") + assert not G.has_edge("dev_poker_server", "tools_aolserver_utility_h_tools_aolserver_utility") + + +def test_build_from_json_unambiguous_old_stem_alias_still_resolves(tmp_path): + """Companion to the ambiguous case above: when exactly one real file claims + an old-stem alias, a dangling edge to that bare alias should still resolve + to it — the #1504 migration-compat behavior this index exists for.""" + root = tmp_path / "repo" + root.mkdir() + extraction = { + "nodes": [ + {"id": "dev_monitoring_utility", "label": "utility.h", "file_type": "code", + "source_file": "Dev/monitoring/utility.h"}, + {"id": "dev_poker_server", "label": "server.cpp", "file_type": "code", + "source_file": "Dev/poker/server.cpp"}, + ], + "edges": [ + {"source": "dev_poker_server", "target": "utility", "relation": "imports", + "confidence": "EXTRACTED", "source_file": "Dev/poker/server.cpp"}, + ], + } + G = build_from_json(extraction, root=root) + assert G.has_edge("dev_poker_server", "dev_monitoring_utility") + + def test_build_from_json_relative_source_file_unchanged(tmp_path): """Already-relative source_file paths must not be modified.""" extraction = { @@ -749,3 +1083,120 @@ def test_semantic_rekey_relative_vs_absolute_source_file(): # absolute path with no resolvable root → skipped, not remapped to an abs-path id ab = [{"id": "api_readme", "source_file": "/abs/docs/v1/api/README.md", "type": "document"}] assert _semantic_id_remap(ab, None) == {} + + +def test_cross_language_imports_references_are_dropped(): + """#1749: an `imports`/`references` edge must not bind across a language + family. A Python `import time` that resolved by bare stem onto a `time.ts` + file node welds the two language halves together at a phantom edge; the spec + forbids this for `calls` and it is equally invalid here.""" + ext = { + "nodes": [ + {"id": "backend_worker_py", "label": "worker.py", "file_type": "code", + "source_file": "backend/worker.py", "source_location": "L1", "_origin": "ast"}, + {"id": "src_time_ts", "label": "time.ts", "file_type": "code", + "source_file": "src/time.ts", "source_location": "L1", "_origin": "ast"}, + {"id": "src_util_ts", "label": "util.ts", "file_type": "code", + "source_file": "src/util.ts", "source_location": "L1", "_origin": "ast"}, + ], + "edges": [ + # phantom: Python file importing a TS file (cross-language) + {"source": "backend_worker_py", "target": "src_time_ts", "relation": "imports", + "confidence": "EXTRACTED", "source_file": "backend/worker.py", "weight": 1.0}, + # legit: TS importing TS (same family) must survive + {"source": "src_time_ts", "target": "src_util_ts", "relation": "imports", + "confidence": "EXTRACTED", "source_file": "src/time.ts", "weight": 1.0}, + ], + } + G = build_from_json(ext, directed=False) + assert not G.has_edge("backend_worker_py", "src_time_ts"), "cross-language import must be dropped" + assert G.has_edge("src_time_ts", "src_util_ts"), "same-family (TS->TS) import must survive" + + +def test_cross_family_reference_to_unknown_ext_is_kept(): + """The #1749 guard only drops when BOTH endpoints are known code languages, + so a reference from a config/manifest (unknown ext) to a code file is kept.""" + ext = { + "nodes": [ + {"id": "pkg_json", "label": "package.json", "file_type": "code", + "source_file": "package.json", "source_location": "L1", "_origin": "ast"}, + {"id": "src_app_ts", "label": "app.ts", "file_type": "code", + "source_file": "src/app.ts", "source_location": "L1", "_origin": "ast"}, + ], + "edges": [ + {"source": "pkg_json", "target": "src_app_ts", "relation": "references", + "confidence": "EXTRACTED", "source_file": "package.json", "weight": 1.0}, + ], + } + G = build_from_json(ext, directed=False) + assert G.has_edge("pkg_json", "src_app_ts"), "config->code reference (unknown ext) must be kept" + + +def test_markdown_doc_twin_merges_into_semantic_doc_node(): + """#1799: the markdown quick-scan's bare `` doc node and the semantic + `_doc` node for the same file must collapse to one node, with edges + consolidated — otherwise a document is two disconnected halves and traversals + dead-end on the wrong twin.""" + ext = { + "nodes": [ + {"id": "docs_readme_doc", "label": "README", "file_type": "document", + "source_file": "docs/readme.md", "source_location": "L1"}, + {"id": "docs_readme", "label": "readme.md", "file_type": "document", + "source_file": "docs/readme.md", "source_location": "L1"}, + {"id": "code_auth", "label": "auth", "file_type": "code", + "source_file": "auth.py", "source_location": "L1"}, + {"id": "docs_guide", "label": "guide.md", "file_type": "document", + "source_file": "docs/guide.md", "source_location": "L1"}, + ], + "edges": [ + {"source": "docs_readme_doc", "target": "code_auth", "relation": "references", + "source_file": "docs/readme.md", "confidence": "INFERRED", "weight": 1.0}, + {"source": "docs_guide", "target": "docs_readme", "relation": "references", + "source_file": "docs/guide.md", "confidence": "EXTRACTED", "weight": 1.0}, + ], + } + G = build_from_json(ext, directed=False) + assert "docs_readme" not in G.nodes() # bare twin merged away + assert "docs_readme_doc" in G.nodes() # semantic node is canonical + assert G.has_edge("docs_guide", "docs_readme_doc") # quick-scan edge repointed + assert G.has_edge("docs_readme_doc", "code_auth") # semantic edge kept + + +def test_doc_twin_merge_does_not_touch_code_symbols(): + """#1799 guard: a code symbol `foo` and an unrelated `foo_doc` (not + file_type=document) must NOT merge, even sharing a source_file.""" + ext = { + "nodes": [ + {"id": "m_foo", "label": "foo", "file_type": "code", + "source_file": "m.py", "source_location": "L1"}, + {"id": "m_foo_doc", "label": "foo rationale", "file_type": "rationale", + "source_file": "m.py", "source_location": "L2"}, + ], + "edges": [], + } + G = build_from_json(ext, directed=False) + assert {"m_foo", "m_foo_doc"} <= set(G.nodes()) + + +def test_build_from_json_prunes_dangling_hyperedge_members(capsys): + """#1916: build_from_json used to copy hyperedges into G.graph["hyperedges"] + verbatim without validating members, so a dangling member reached graph.json + even from a live (non-cache) extraction. Members absent from the built node + set are pruned — matching how dangling pairwise edges are skipped — and a + hyperedge with no surviving member is dropped whole.""" + ext = { + "nodes": [ + {"id": "alpha", "label": "alpha", "file_type": "code", "source_file": "a.py"}, + {"id": "beta", "label": "beta", "file_type": "code", "source_file": "a.py"}, + ], + "edges": [], + "hyperedges": [ + {"id": "he_partial", "nodes": ["alpha", "beta", "ghost_member"], "source_file": "a.py"}, + {"id": "he_all_ghost", "nodes": ["ghost1", "ghost2"], "source_file": "a.py"}, + ], + } + G = build_from_json(ext) + hes = {h["id"]: h for h in G.graph.get("hyperedges", [])} + assert set(hes) == {"he_partial"}, "an all-dangling hyperedge must be dropped" + assert hes["he_partial"]["nodes"] == ["alpha", "beta"] + assert "he_all_ghost" in capsys.readouterr().err diff --git a/tests/test_build_merge_hyperedges_and_prune.py b/tests/test_build_merge_hyperedges_and_prune.py index ff4c7e8f6..d3629a73c 100644 --- a/tests/test_build_merge_hyperedges_and_prune.py +++ b/tests/test_build_merge_hyperedges_and_prune.py @@ -149,3 +149,118 @@ def test_prune_matches_across_symlinked_root(tmp_path): prune_sources=[str(link / "HANDOFF.md")], root=str(real), dedup=False) labels = {d["label"] for _, d in G.nodes(data=True)} assert "handoff" not in labels and "keep" in labels + + +def test_reextracted_file_in_prune_sources_is_not_deleted(tmp_path): + """#1796: a file present in BOTH new_chunks (re-extracted) and prune_sources + must be REPLACED, not deleted. The old edit-workflow passed the changed file + in prune_sources; combined with dedup keeping a same-label node, that used to + silently delete the freshly re-extracted concept. Replace wins over delete.""" + graph_path = tmp_path / "graphify-out" / "graph.json" + graph_path.parent.mkdir(parents=True) + _write_graph( + graph_path, + nodes=[ + {"id": "foo_widget_cache", "label": "Widget Cache Design", + "file_type": "concept", "source_file": "docs/foo.md", "source_location": "L1"}, + {"id": "bar_other", "label": "Other", + "file_type": "concept", "source_file": "docs/bar.md", "source_location": "L1"}, + ], + edges=[], + hyperedges=[], + ) + # foo.md edited: same-label node re-extracted (new content/line) + new_chunk = {"nodes": [ + {"id": "foo_widget_cache", "label": "Widget Cache Design", + "file_type": "concept", "source_file": "docs/foo.md", "source_location": "L2"} + ], "edges": []} + + G = build_merge([new_chunk], graph_path=str(graph_path), + prune_sources=["docs/foo.md"], root=str(tmp_path)) + labels = {G.nodes[n].get("label") for n in G.nodes()} + assert "Widget Cache Design" in labels, "re-extracted node was wrongly pruned" + + +def test_genuine_deletion_still_prunes(tmp_path): + """#1796 guard must not break real deletions: a file in prune_sources but NOT + in new_chunks is still removed.""" + graph_path = tmp_path / "graphify-out" / "graph.json" + graph_path.parent.mkdir(parents=True) + _write_graph( + graph_path, + nodes=[ + {"id": "foo_widget_cache", "label": "Widget Cache Design", + "file_type": "concept", "source_file": "docs/foo.md", "source_location": "L1"}, + {"id": "bar_other", "label": "Other", + "file_type": "concept", "source_file": "docs/bar.md", "source_location": "L1"}, + ], + edges=[], + hyperedges=[], + ) + new_chunk = {"nodes": [ + {"id": "foo_widget_cache", "label": "Widget Cache Design", + "file_type": "concept", "source_file": "docs/foo.md", "source_location": "L2"} + ], "edges": []} + # bar.md genuinely deleted (not re-extracted) + G = build_merge([new_chunk], graph_path=str(graph_path), + prune_sources=["docs/bar.md"], root=str(tmp_path)) + labels = {G.nodes[n].get("label") for n in G.nodes()} + assert "Other" not in labels, "genuinely deleted file's node should be pruned" + assert "Widget Cache Design" in labels + + +# ── #2012: form-insensitive prune (absolute node vs relative prune, and back) ── + +def test_prune_matches_node_stored_absolute_against_relative_delete(tmp_path): + """#2012: a node whose source_file survived in ABSOLUTE form must still be + pruned when the deletion is expressed relative to root. The runbook calls + build_merge WITHOUT root, so build() does not re-normalize the node's stored + absolute source_file; the old prune membership test then compared that raw + absolute string against a prune_set that only held the relative forms, so the + node slipped through and a deleted file's graph survived silently. build_merge + now normalizes the node side too + an absolute-identity fallback.""" + root = tmp_path / "corpus" + (root / "graphify-out").mkdir(parents=True) + graph_path = root / "graphify-out" / "graph.json" + nodes = [ + # gone.py's node kept an ABSOLUTE source_file (a semantic subagent wrote + # it that way, #932); keep.py's is relative. + {"id": "g1", "label": "gone", "file_type": "code", + "source_file": str(root / "gone.py")}, + {"id": "k1", "label": "keep", "file_type": "code", "source_file": "keep.py"}, + ] + edges = [ + {"source": "g1", "target": "k1", "type": "calls", + "source_file": str(root / "gone.py")}, + ] + _write_graph(graph_path, nodes, edges, []) + # Runbook-style: NO root passed (eff_root inferred from the graphify-out + # grandparent), so build() leaves the absolute node form intact. Deletion is + # expressed RELATIVE — a third form vs the stored absolute node. + G = build_merge([], graph_path, prune_sources=["gone.py"], dedup=False) + labels = {d["label"] for _, d in G.nodes(data=True)} + assert "gone" not in labels, "absolute-stored node not pruned by relative delete (#2012)" + assert "keep" in labels + assert G.number_of_edges() == 0, "edge from the deleted file must be pruned too (#2012)" + + +def test_prune_reextracted_absolute_node_not_deleted(tmp_path): + """#1796 protection must hold in absolute-identity space too: a file present + in BOTH new_chunks and prune_sources (in mismatched forms) is REPLACED, not + deleted — the #2012 form-insensitive match must not resurrect the delete for + a re-extracted file.""" + root = tmp_path / "corpus" + (root / "graphify-out").mkdir(parents=True) + graph_path = root / "graphify-out" / "graph.json" + _write_graph(graph_path, [ + {"id": "g1", "label": "gone", "file_type": "code", + "source_file": str(root / "mod.py")}, + ], [], []) + # Re-extracted with a RELATIVE source_file; prune lists it RELATIVE too. + # No root passed (runbook), so the stored absolute node is not re-normalized. + new_chunk = {"nodes": [ + {"id": "g1", "label": "gone", "file_type": "code", "source_file": "mod.py"}, + ], "edges": []} + G = build_merge([new_chunk], graph_path, prune_sources=["mod.py"], dedup=False) + labels = {d["label"] for _, d in G.nodes(data=True)} + assert "gone" in labels, "re-extracted file wrongly pruned across mismatched forms (#2012/#1796)" diff --git a/tests/test_builtin_global_type_refs.py b/tests/test_builtin_global_type_refs.py new file mode 100644 index 000000000..25ecb151d --- /dev/null +++ b/tests/test_builtin_global_type_refs.py @@ -0,0 +1,83 @@ +"""Builtin-global receiver types must not resolve to same-named user symbols. + +#1726: `x: Date; x.getTime()` had its caller bound (by casefolded label) to a +user `class DATE` / `const DATE` in another file, inventing phantom +`references[call]` edges and a false god node. The cross-file CALL resolver +already skips ECMAScript/Python builtins; `_resolve_typescript_member_calls` +must do the same. +""" +from pathlib import Path +from graphify.extract import extract + + +def _labels_by_id(r): + return {n["id"]: n.get("label") for n in r["nodes"]} + + +def test_builtin_date_type_ref_does_not_bind_to_user_DATE(tmp_path): + (tmp_path / "model.ts").write_text('export class DATE {\n value: string = "";\n}\n') + (tmp_path / "a.ts").write_text('export function parse(x: Date): number { return x.getTime(); }\n') + (tmp_path / "b.ts").write_text('export function fmt(w: Date): string { return w.toISOString(); }\n') + r = extract(sorted(tmp_path.glob("*.ts")), cache_root=tmp_path, parallel=False) + lbl = _labels_by_id(r) + date_ids = [n["id"] for n in r["nodes"] if n.get("label") == "DATE"] + assert date_ids, "the user class DATE must still exist as a node" + for e in r["edges"]: + if e.get("relation") == "references" and e.get("target") in date_ids: + src = lbl.get(e["source"]) + assert False, f"phantom builtin-Date reference bound to user DATE from {src!r}" + # the user DATE node accumulates no phantom references — degree is just its file + deg = sum(1 for e in r["edges"] if date_ids[0] in (e["source"], e["target"])) + assert deg <= 1, f"user DATE should not be a god node; degree={deg}" + + +def test_nonbuiltin_receiver_type_still_resolves(tmp_path): + # Guard must be a no-op for a genuine user type: a member call on a user-typed + # field still resolves cross-file (constructor-injection type table, #1316). + (tmp_path / "svc.ts").write_text( + "export class PaymentClient {\n charge(n: number): boolean { return true; }\n}\n") + (tmp_path / "order.ts").write_text( + 'import { PaymentClient } from "./svc";\n' + "export class Order {\n" + " constructor(private client: PaymentClient) {}\n" + " pay(): boolean { return this.client.charge(1); }\n" + "}\n") + r = extract(sorted(tmp_path.glob("*.ts")), cache_root=tmp_path, parallel=False) + lbl = _labels_by_id(r) + resolved = { + (lbl.get(e["source"]), lbl.get(e["target"]), e["relation"]) + for e in r["edges"] if "charge" in str(e.get("target", "")).lower() + } + assert any(t and "charge" in str(t).lower() for _, t, _ in resolved), \ + f"user member-call must still resolve; got {resolved}" + + +def test_builtin_static_call_does_not_bind_to_user_symbol(tmp_path): + # #1726 (static-call shape, credit PR #1727 / @2loch-ness6): `Date.now()` treats + # the capitalized receiver `Date` as a type name; without the builtin guard it + # binds to a same-spelled user `const DATE`, a false god node. A typed param in + # the same class arms the cross-file member-call resolver (the real service shape). + (tmp_path / "format.ts").write_text( + "const DATE = new Intl.DateTimeFormat('en-US', {});\n" + "export function fmt(x: number): string { return DATE.format(x); }\n") + (tmp_path / "svc.ts").write_text( + "export class Svc {\n" + " expiry(d: Date): Date { return d; }\n" + " stamp(): number { return Date.now(); }\n" + " when(): string { return new Date().toISOString(); }\n" + "}\n") + r = extract(sorted(tmp_path.glob("*.ts")), cache_root=tmp_path, parallel=False) + lbl = _labels_by_id(r) + by_id = {n["id"]: n for n in r["nodes"]} + date_ids = [n["id"] for n in r["nodes"] if n.get("label") == "DATE"] + date_sf = {str(by_id[i].get("source_file", "")) for i in date_ids} + # A same-file reference to the real const DATE (fmt() -> DATE in format.ts) is + # legitimate. The bug is a CROSS-FILE bind: svc.ts's Date.now()/new Date() + # resolving to format.ts's const DATE. Assert no such cross-file phantom. + for e in r["edges"]: + if e.get("target") in date_ids and e.get("relation") == "references": + src_sf = str(by_id.get(e["source"], {}).get("source_file", "")) + assert src_sf in date_sf, ( + f"cross-file builtin phantom: {lbl.get(e['source'])!r} in {src_sf} " + f"bound to user DATE" + ) diff --git a/tests/test_cache.py b/tests/test_cache.py index 730265be4..993c5a0d2 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -510,3 +510,627 @@ def test_semantic_prune_ignores_ast_and_tmp(tmp_path): assert not (semantic_dir / "deadbeef.json").exists() assert tmp_entry.exists(), "*.tmp temporaries must not be swept" assert len(list(ast_dir.glob("*.json"))) == 1, "AST entries must not be touched" + + +def test_save_semantic_cache_overwrites_by_default(tmp_path): + """Default save_semantic_cache replaces a file's cached entry (the final, + authoritative write in the extract pipeline).""" + from graphify.cache import save_semantic_cache + f = tmp_path / "doc.md"; f.write_text("# Doc\n") + save_semantic_cache([{"id": "a", "source_file": "doc.md"}], [], root=tmp_path) + save_semantic_cache([{"id": "b", "source_file": "doc.md"}], [], root=tmp_path) + cached = load_cached(f, root=tmp_path, kind="semantic") + ids = {n["id"] for n in cached["nodes"]} + assert ids == {"b"}, "default must overwrite, not accumulate" + + +def test_save_semantic_cache_rejects_out_of_scope_source_file(tmp_path): + """#1757: an undispatched file must keep its complete cache entry when a + semantic result misattributes a node to it.""" + from graphify.cache import save_semantic_cache + + intended = tmp_path / "intended.md" + intended.write_text("# Intended\n") + protected = tmp_path / "protected.md" + protected.write_text("# Protected\n") + + save_semantic_cache( + [{"id": "original", "source_file": "protected.md"}], + [], + root=tmp_path, + ) + + nodes = [ + {"id": "expected", "source_file": str(intended.resolve())}, + {"id": "stray", "source_file": "protected.md"}, + ] + edges = [ + {"source": "stray", "target": "expected", "source_file": "protected.md"}, + ] + hyperedges = [ + {"id": "stray_hyperedge", "nodes": ["stray"], "source_file": "protected.md"}, + ] + + with pytest.warns(RuntimeWarning, match="out-of-scope source_file 'protected.md'"): + saved = save_semantic_cache( + nodes, + edges, + hyperedges, + root=tmp_path, + allowed_source_files=["intended.md"], + ) + + assert saved == 1 + intended_cache = load_cached(intended, root=tmp_path, kind="semantic") + assert {node["id"] for node in intended_cache["nodes"]} == {"expected"} + + protected_cache = load_cached(protected, root=tmp_path, kind="semantic") + assert {node["id"] for node in protected_cache["nodes"]} == {"original"} + assert protected_cache["edges"] == [] + assert protected_cache["hyperedges"] == [] + + +# --- #1894: mode-namespaced semantic cache ----------------------------------- +# `extract --mode deep` produces richer results than standard extraction, so +# deep entries live in their own namespace (cache/semantic-deep/). mode=None +# must stay byte-identical to the historical behavior: older installed skill +# flows call check/save without the parameter and must be unaffected. + +def test_semantic_cache_deep_mode_roundtrip_under_deep_namespace(tmp_path): + """mode='deep' saves under cache/semantic-deep/ and reads back from it.""" + from graphify.cache import check_semantic_cache, save_semantic_cache + + f = tmp_path / "doc.md" + f.write_text("# Doc\n\nBody.\n") + saved = save_semantic_cache( + [{"id": "deep_n", "source_file": "doc.md"}], [], root=tmp_path, mode="deep" + ) + assert saved == 1 + + deep_dir = tmp_path / "graphify-out" / "cache" / "semantic-deep" + h = file_hash(f, tmp_path) + assert (deep_dir / f"{h}.json").exists(), ( + "deep entry must land under cache/semantic-deep/" + ) + # And NOT in the plain namespace. + plain_dir = tmp_path / "graphify-out" / "cache" / "semantic" + assert not (plain_dir / f"{h}.json").exists() + + nodes, edges, hyper, uncached = check_semantic_cache( + [str(f)], root=tmp_path, mode="deep" + ) + assert [n["id"] for n in nodes] == ["deep_n"] + assert uncached == [] + + +def test_semantic_cache_deep_invisible_to_plain_reads_and_vice_versa(tmp_path): + """Deep entries must not satisfy mode=None reads (and plain entries must + not satisfy deep reads) — the namespaces are fully isolated.""" + from graphify.cache import check_semantic_cache, save_semantic_cache + + deep_doc = tmp_path / "deep.md" + deep_doc.write_text("# Deep\n") + plain_doc = tmp_path / "plain.md" + plain_doc.write_text("# Plain\n") + + save_semantic_cache([{"id": "d", "source_file": "deep.md"}], [], + root=tmp_path, mode="deep") + save_semantic_cache([{"id": "p", "source_file": "plain.md"}], [], + root=tmp_path) # mode omitted: historical call shape + + # Plain read: deep entry is a miss, plain entry is a hit. + nodes, _, _, uncached = check_semantic_cache( + [str(deep_doc), str(plain_doc)], root=tmp_path + ) + assert [n["id"] for n in nodes] == ["p"] + assert uncached == [str(deep_doc)] + + # Deep read: mirror image. + nodes, _, _, uncached = check_semantic_cache( + [str(deep_doc), str(plain_doc)], root=tmp_path, mode="deep" + ) + assert [n["id"] for n in nodes] == ["d"] + assert uncached == [str(plain_doc)] + + +def test_semantic_cache_mode_none_layout_unchanged(tmp_path): + """Omitting mode writes exactly the historical cache/semantic/ layout — + forward-compat for older installed callers that never pass mode.""" + from graphify.cache import check_semantic_cache, save_semantic_cache + + f = tmp_path / "doc.md" + f.write_text("# Doc\n") + save_semantic_cache([{"id": "n", "source_file": "doc.md"}], [], root=tmp_path) + h = file_hash(f, tmp_path) + assert (tmp_path / "graphify-out" / "cache" / "semantic" / f"{h}.json").exists() + assert not (tmp_path / "graphify-out" / "cache" / "semantic-deep").exists(), ( + "mode=None must never create the deep namespace" + ) + nodes, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path) + assert [n["id"] for n in nodes] == ["n"] and uncached == [] + + +def test_clear_cache_removes_deep_namespace(tmp_path): + """clear_cache sweeps cache/semantic-deep/ alongside semantic/ and ast/.""" + from graphify.cache import save_semantic_cache + + f = tmp_path / "doc.md" + f.write_text("# Doc\n") + save_semantic_cache([{"id": "p", "source_file": "doc.md"}], [], root=tmp_path) + save_semantic_cache([{"id": "d", "source_file": "doc.md"}], [], + root=tmp_path, mode="deep") + base = tmp_path / "graphify-out" / "cache" + assert list((base / "semantic").glob("*.json")) + assert list((base / "semantic-deep").glob("*.json")) + + clear_cache(tmp_path) + assert not list(base.rglob("*.json")), ( + "clear_cache must remove entries in BOTH semantic namespaces" + ) + + +def test_cached_files_includes_deep_namespace(tmp_path): + """cached_files reports deep-namespace entries too.""" + from graphify.cache import save_semantic_cache + + f = tmp_path / "doc.md" + f.write_text("# Doc\n") + save_semantic_cache([{"id": "d", "source_file": "doc.md"}], [], + root=tmp_path, mode="deep") + assert file_hash(f, tmp_path) in cached_files(tmp_path) + + +def test_semantic_prune_sweeps_both_namespaces_against_same_live_set(tmp_path): + """#1894 follow-up to #1527: prune must sweep cache/semantic/ AND + cache/semantic-deep/ against the SAME live-hash set (liveness is + content-based, mode-independent). Orphans go in both namespaces; live + entries survive in both.""" + from graphify.cache import prune_semantic_cache, save_semantic_cache + + f = tmp_path / "doc.md" + f.write_text("# A\n\nContent A.\n") + h_old = file_hash(f, tmp_path) + save_semantic_cache([{"id": "pa", "source_file": "doc.md"}], [], root=tmp_path) + save_semantic_cache([{"id": "da", "source_file": "doc.md"}], [], + root=tmp_path, mode="deep") + + f.write_text("# B\n\nContent B.\n") + h_live = file_hash(f, tmp_path) + save_semantic_cache([{"id": "pb", "source_file": "doc.md"}], [], root=tmp_path) + save_semantic_cache([{"id": "db", "source_file": "doc.md"}], [], + root=tmp_path, mode="deep") + + plain_dir = tmp_path / "graphify-out" / "cache" / "semantic" + deep_dir = tmp_path / "graphify-out" / "cache" / "semantic-deep" + for d in (plain_dir, deep_dir): + assert (d / f"{h_old}.json").exists() + assert (d / f"{h_live}.json").exists() + + pruned = prune_semantic_cache(tmp_path, {h_live}) + assert pruned == 2, "one orphan in EACH namespace must be pruned" + for d in (plain_dir, deep_dir): + assert not (d / f"{h_old}.json").exists(), f"orphan survived in {d.name}" + assert (d / f"{h_live}.json").exists(), f"live entry pruned from {d.name}" + + +def test_save_semantic_cache_merge_existing_unions(tmp_path): + """#1715: merge_existing=True unions with the prior entry so a file split + across chunks (checkpointed per chunk) keeps every slice.""" + from graphify.cache import save_semantic_cache + f = tmp_path / "big.md"; f.write_text("# Big\n") + # chunk 1 slice + save_semantic_cache([{"id": "a", "source_file": "big.md"}], + [{"source": "a", "target": "x", "source_file": "big.md"}], + root=tmp_path, merge_existing=True) + # chunk 2 slice for the same file + save_semantic_cache([{"id": "b", "source_file": "big.md"}], [], + root=tmp_path, merge_existing=True) + cached = load_cached(f, root=tmp_path, kind="semantic") + ids = {n["id"] for n in cached["nodes"]} + assert ids == {"a", "b"}, "merge_existing must union both chunk slices" + assert len(cached["edges"]) == 1 + + +def test_save_semantic_cache_drops_edges_to_out_of_scope_nodes(tmp_path): + """#1916: an edge in an ALLOWED file's group referencing a node grouped + under an out-of-scope REAL file used to be written verbatim, so on replay + (check_semantic_cache) it dangled forever — the #1895 merged-result filter + runs after this checkpoint write and is bypassed entirely on replay. The + written entry must carry no reference to the skipped id, while a + duplicate-attribution node (also defined in a written group) must not be + over-pruned.""" + from graphify.cache import check_semantic_cache, save_semantic_cache + + allowed = tmp_path / "allowed.md" + allowed.write_text("# Allowed\n") + outside = tmp_path / "outside.md" + outside.write_text("# Outside\n") + + nodes = [ + {"id": "kept", "source_file": "allowed.md"}, + {"id": "stray", "source_file": "outside.md"}, + # duplicate attribution: same id defined in a written AND a skipped group + {"id": "dup", "source_file": "allowed.md"}, + {"id": "dup", "source_file": "outside.md"}, + ] + edges = [ + {"source": "kept", "target": "stray", "source_file": "allowed.md"}, + {"source": "stray", "target": "kept", "source_file": "allowed.md"}, + {"source": "kept", "target": "dup", "source_file": "allowed.md"}, + ] + with pytest.warns(RuntimeWarning, match="out-of-scope source_file"): + saved = save_semantic_cache( + nodes, edges, root=tmp_path, allowed_source_files=["allowed.md"] + ) + assert saved == 1 + + cached_nodes, cached_edges, _, uncached = check_semantic_cache( + [str(allowed)], root=tmp_path + ) + assert uncached == [] + assert {n["id"] for n in cached_nodes} == {"kept", "dup"} + pairs = [(e["source"], e["target"]) for e in cached_edges] + assert pairs == [("kept", "dup")], "edges touching the skipped id must be dropped" + + +def test_save_semantic_cache_drops_edges_to_ghost_file_nodes(tmp_path): + """#1916 (ghost variant): a node group whose source_file does not exist is + silently skipped by the write loop; edges in a written group referencing + its node ids must not survive into the cache.""" + from graphify.cache import check_semantic_cache, save_semantic_cache + + real = tmp_path / "real.md" + real.write_text("# Real\n") + + nodes = [ + {"id": "kept", "source_file": "real.md"}, + {"id": "phantom", "source_file": "ghost.md"}, # no such file on disk + ] + edges = [ + {"source": "kept", "target": "phantom", "source_file": "real.md"}, + {"source": "kept", "target": "kept", "relation": "self", "source_file": "real.md"}, + ] + saved = save_semantic_cache( + nodes, edges, root=tmp_path, allowed_source_files=["real.md"] + ) + assert saved == 1 + + cached_nodes, cached_edges, _, uncached = check_semantic_cache( + [str(real)], root=tmp_path + ) + assert uncached == [] + assert {n["id"] for n in cached_nodes} == {"kept"} + pairs = [(e["source"], e["target"]) for e in cached_edges] + assert pairs == [("kept", "kept")] + + +def test_save_semantic_cache_drops_hyperedges_touching_skipped_nodes(tmp_path): + """#1916: a hyperedge whose member list intersects the skipped ids is + dropped whole (mirroring the #1895 semantics), while hyperedges over + surviving nodes are kept.""" + from graphify.cache import check_semantic_cache, save_semantic_cache + + allowed = tmp_path / "allowed.md" + allowed.write_text("# Allowed\n") + outside = tmp_path / "outside.md" + outside.write_text("# Outside\n") + + nodes = [ + {"id": "kept", "source_file": "allowed.md"}, + {"id": "kept2", "source_file": "allowed.md"}, + {"id": "stray", "source_file": "outside.md"}, + ] + hyperedges = [ + {"id": "he_bad", "nodes": ["kept", "stray"], "source_file": "allowed.md"}, + {"id": "he_ok", "nodes": ["kept", "kept2"], "source_file": "allowed.md"}, + ] + with pytest.warns(RuntimeWarning, match="out-of-scope source_file"): + save_semantic_cache( + nodes, [], hyperedges, root=tmp_path, allowed_source_files=["allowed.md"] + ) + + _, _, cached_hyperedges, uncached = check_semantic_cache( + [str(allowed)], root=tmp_path + ) + assert uncached == [] + assert {h["id"] for h in cached_hyperedges} == {"he_ok"} + + +def test_save_semantic_cache_unscoped_preserves_dangling_refs_verbatim(tmp_path): + """#1916 guard-rail: unscoped callers (allowed_source_files=None) must stay + byte-identical — no pruning happens even when an edge or hyperedge + references a node grouped under a ghost file.""" + from graphify.cache import save_semantic_cache + + doc = tmp_path / "doc.md" + doc.write_text("# Doc\n") + + nodes = [ + {"id": "a", "source_file": "doc.md"}, + {"id": "ghost_n", "source_file": "ghost.md"}, # skipped group (no file) + ] + edges = [{"source": "a", "target": "ghost_n", "source_file": "doc.md"}] + hyperedges = [{"id": "he", "nodes": ["a", "ghost_n"], "source_file": "doc.md"}] + + saved = save_semantic_cache(nodes, edges, hyperedges, root=tmp_path) + assert saved == 1 + + import json + raw = json.loads( + (cache_dir(tmp_path, "semantic") / f"{file_hash(doc, tmp_path)}.json").read_text() + ) + assert raw["edges"] == edges + assert raw["hyperedges"] == hyperedges + + +def test_save_semantic_cache_merge_existing_prunes_only_incoming(tmp_path): + """#1916 + #1715: with merge_existing=True (the llm.py checkpoint path), + only the INCOMING slice is pruned before the union — the prior cached + entry's valid edges must survive untouched.""" + from graphify.cache import save_semantic_cache + + big = tmp_path / "big.md" + big.write_text("# Big\n") + other = tmp_path / "other.md" + other.write_text("# Other\n") + + # checkpoint 1: a clean slice + save_semantic_cache( + [{"id": "a", "source_file": "big.md"}], + [{"source": "a", "target": "a", "relation": "self", "source_file": "big.md"}], + root=tmp_path, + merge_existing=True, + allowed_source_files=["big.md"], + ) + # checkpoint 2: incoming slice with a dangling edge to an out-of-scope node + nodes2 = [ + {"id": "b", "source_file": "big.md"}, + {"id": "stray", "source_file": "other.md"}, + ] + edges2 = [ + {"source": "b", "target": "stray", "source_file": "big.md"}, + {"source": "a", "target": "b", "source_file": "big.md"}, + ] + with pytest.warns(RuntimeWarning, match="out-of-scope source_file"): + save_semantic_cache( + nodes2, edges2, root=tmp_path, merge_existing=True, + allowed_source_files=["big.md"], + ) + + cached = load_cached(big, root=tmp_path, kind="semantic") + assert {n["id"] for n in cached["nodes"]} == {"a", "b"} + pairs = [(e["source"], e["target"]) for e in cached["edges"]] + assert ("a", "a") in pairs, "prior entry's valid edge must survive the union" + assert ("a", "b") in pairs, "incoming valid edge must be kept" + assert not any("stray" in p for p in pairs) + + +# --- extraction-prompt fingerprinting (#1939) ------------------------------- + + +def test_prompt_fingerprint_stable_and_prompt_sensitive(tmp_path): + """The fingerprint is stable for identical prompts and differs when the + prompt text changes — the whole invalidation signal rests on this.""" + from graphify.cache import prompt_fingerprint + + assert prompt_fingerprint("extract a graph") == prompt_fingerprint("extract a graph") + assert prompt_fingerprint("extract a graph") != prompt_fingerprint("extract a graph v2") + + # A Path is read and hashed as its contents, so the skill path (which loads + # references/extraction-spec.md) and the Python path agree on the same text. + spec = tmp_path / "extraction-spec.md" + spec.write_text("extract a graph", encoding="utf-8") + assert prompt_fingerprint(spec) == prompt_fingerprint("extract a graph") + + +def test_prompt_fingerprint_ignores_line_endings(tmp_path): + """A CRLF checkout of the same spec must not look like a prompt change — + otherwise every Windows run re-bills the whole corpus.""" + from graphify.cache import prompt_fingerprint + + assert prompt_fingerprint("a\r\nb\r\n") == prompt_fingerprint("a\nb\n") + assert prompt_fingerprint("a \nb\n") == prompt_fingerprint("a\nb\n") + + +def test_semantic_cache_prompt_change_invalidates(tmp_path): + """The reported bug (#1939): after the extraction prompt changes, an + unchanged file must MISS instead of replaying the older vintage.""" + from graphify.cache import check_semantic_cache, save_semantic_cache + + f = tmp_path / "doc.md" + f.write_text("# Doc\n\nBody.\n") + save_semantic_cache([{"id": "old_vintage", "source_file": "doc.md"}], [], + root=tmp_path, prompt="PROMPT V1") + + # Same prompt: hit. + nodes, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path, prompt="PROMPT V1") + assert [n["id"] for n in nodes] == ["old_vintage"] + assert uncached == [] + + # Prompt changed (an upgrade shipped a new extraction-spec): must re-extract. + nodes, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path, prompt="PROMPT V2") + assert nodes == [] + assert uncached == [str(f)], "a new prompt must not replay the old prompt's entry" + + # V2's results land in their own namespace and do not clobber V1's, so + # rolling back to V1 still hits rather than re-billing. + save_semantic_cache([{"id": "new_vintage", "source_file": "doc.md"}], [], + root=tmp_path, prompt="PROMPT V2") + nodes, _, _, _ = check_semantic_cache([str(f)], root=tmp_path, prompt="PROMPT V2") + assert [n["id"] for n in nodes] == ["new_vintage"] + nodes, _, _, _ = check_semantic_cache([str(f)], root=tmp_path, prompt="PROMPT V1") + assert [n["id"] for n in nodes] == ["old_vintage"] + + +def test_semantic_cache_prompt_namespaced_layout(tmp_path): + """Fingerprinted entries live under cache/semantic/p{fp}/, never flat.""" + from graphify.cache import prompt_fingerprint, save_semantic_cache + + f = tmp_path / "doc.md" + f.write_text("# Doc\n") + save_semantic_cache([{"id": "n", "source_file": "doc.md"}], [], + root=tmp_path, prompt="PROMPT V1") + + sem = tmp_path / "graphify-out" / "cache" / "semantic" + h = file_hash(f, tmp_path) + assert (sem / f"p{prompt_fingerprint('PROMPT V1')}" / f"{h}.json").exists() + assert not (sem / f"{h}.json").exists(), ( + "a known-vintage entry must never be written into the flat unknown-vintage layout" + ) + + +def test_semantic_cache_prompt_and_mode_compose(tmp_path): + """The prompt fingerprint nests inside the deep namespace (#1894), so the + two dimensions are independent.""" + from graphify.cache import check_semantic_cache, save_semantic_cache + + f = tmp_path / "doc.md" + f.write_text("# Doc\n") + save_semantic_cache([{"id": "d", "source_file": "doc.md"}], [], + root=tmp_path, mode="deep", prompt="PROMPT V1") + + deep = tmp_path / "graphify-out" / "cache" / "semantic-deep" + assert list(deep.glob("p*/*.json")), "deep + prompt must nest under semantic-deep/p{fp}/" + + # Right mode, wrong prompt -> miss. Right prompt, wrong mode -> miss. + _, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path, mode="deep", + prompt="PROMPT V2") + assert uncached == [str(f)] + _, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path, prompt="PROMPT V1") + assert uncached == [str(f)] + # Both right -> hit. + nodes, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path, mode="deep", + prompt="PROMPT V1") + assert [n["id"] for n in nodes] == ["d"] and uncached == [] + + +def test_semantic_cache_legacy_entries_served_with_warning(tmp_path): + """Entries written before fingerprinting have unknowable vintage. They are + still served — dropping them would re-bill a whole corpus on upgrade — but + the user is told how many, which is the signal #1939 says is missing today.""" + from graphify.cache import check_semantic_cache, save_semantic_cache + + a = tmp_path / "a.md" + a.write_text("# A\n") + b = tmp_path / "b.md" + b.write_text("# B\n") + # Pre-fingerprint writes: the historical flat layout. + save_semantic_cache([{"id": "a_old", "source_file": "a.md"}, + {"id": "b_old", "source_file": "b.md"}], [], root=tmp_path) + + with pytest.warns(RuntimeWarning, match="2 semantic cache entries predate"): + nodes, _, _, uncached = check_semantic_cache( + [str(a), str(b)], root=tmp_path, prompt="PROMPT V1" + ) + assert {n["id"] for n in nodes} == {"a_old", "b_old"} + assert uncached == [] + + +def test_semantic_cache_fingerprinted_entry_beats_legacy(tmp_path): + """Once a file is re-extracted under the current prompt, its fingerprinted + entry wins and the stale flat one is no longer consulted (no warning).""" + import warnings as _warnings + from graphify.cache import check_semantic_cache, save_semantic_cache + + f = tmp_path / "doc.md" + f.write_text("# Doc\n") + save_semantic_cache([{"id": "unknown_vintage", "source_file": "doc.md"}], [], + root=tmp_path) # legacy flat + save_semantic_cache([{"id": "current", "source_file": "doc.md"}], [], + root=tmp_path, prompt="PROMPT V1") + + with _warnings.catch_warnings(): + _warnings.simplefilter("error") # any legacy warning would raise here + nodes, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path, + prompt="PROMPT V1") + assert [n["id"] for n in nodes] == ["current"] + assert uncached == [] + + +def test_semantic_cache_merge_existing_never_fuses_legacy_vintage(tmp_path): + """merge_existing must not union a pre-fingerprint entry into a write it is + about to stamp as current-vintage — that would mix two prompts inside one + entry and then attest the result to a prompt that produced half of it.""" + from graphify.cache import load_cached, save_semantic_cache + + f = tmp_path / "doc.md" + f.write_text("# Doc\n") + save_semantic_cache([{"id": "unknown_vintage", "source_file": "doc.md"}], [], + root=tmp_path) # legacy flat + save_semantic_cache([{"id": "current", "source_file": "doc.md"}], [], + root=tmp_path, merge_existing=True, prompt="PROMPT V1") + + entry = load_cached(f, root=tmp_path, kind="semantic", prompt="PROMPT V1") + assert [n["id"] for n in entry["nodes"]] == ["current"] + + # Within one prompt, merge_existing still unions across checkpoints. + save_semantic_cache([{"id": "second_chunk", "source_file": "doc.md"}], [], + root=tmp_path, merge_existing=True, prompt="PROMPT V1") + entry = load_cached(f, root=tmp_path, kind="semantic", prompt="PROMPT V1") + assert {n["id"] for n in entry["nodes"]} == {"current", "second_chunk"} + + +def test_semantic_prune_and_clear_reach_fingerprint_subdirs(tmp_path): + """A glob that stopped at the top level would leave every fingerprinted + entry unprunable, re-growing the unbounded-orphan problem of #1527.""" + from graphify.cache import ( + cached_files, clear_cache, prune_semantic_cache, save_semantic_cache, + ) + + f = tmp_path / "doc.md" + f.write_text("# Doc\n") + save_semantic_cache([{"id": "n", "source_file": "doc.md"}], [], + root=tmp_path, prompt="PROMPT V1") + h = file_hash(f, tmp_path) + assert h in cached_files(tmp_path), "cached_files must see fingerprinted entries" + + # Live: kept. + assert prune_semantic_cache(tmp_path, {h}) == 0 + # Orphaned (content changed / file deleted): pruned. + assert prune_semantic_cache(tmp_path, set()) == 1 + + save_semantic_cache([{"id": "n", "source_file": "doc.md"}], [], + root=tmp_path, prompt="PROMPT V1") + clear_cache(tmp_path) + assert not list((tmp_path / "graphify-out" / "cache" / "semantic").glob("**/*.json")) + + +def test_semantic_cache_unreadable_prompt_file_warns_and_falls_back(tmp_path): + """A skill snippet substitutes SPEC_PATH by hand. If it lands on a path that + isn't there, the fallback to the unattributed layout must be loud: silently + reverting to unversioned keying is exactly the #1939 behavior being fixed.""" + from graphify.cache import check_semantic_cache, save_semantic_cache + + f = tmp_path / "doc.md" + f.write_text("# Doc\n") + save_semantic_cache([{"id": "n", "source_file": "doc.md"}], [], root=tmp_path) + + with pytest.warns(RuntimeWarning, match="could not read extraction prompt"): + nodes, _, _, uncached = check_semantic_cache( + [str(f)], root=tmp_path, prompt_file=str(tmp_path / "nope.md") + ) + # Fell back rather than aborting the run. + assert [n["id"] for n in nodes] == ["n"] and uncached == [] + + +def test_prompt_file_reflects_edited_spec(tmp_path): + """The prompt-file fingerprint is memoized per (path, size, mtime); an edited + spec must still register as a new prompt rather than reusing a stale memo.""" + from graphify.cache import check_semantic_cache, save_semantic_cache + + spec = tmp_path / "extraction-spec.md" + spec.write_text("prompt one", encoding="utf-8") + f = tmp_path / "doc.md" + f.write_text("# Doc\n") + + save_semantic_cache([{"id": "v1", "source_file": "doc.md"}], [], + root=tmp_path, prompt_file=str(spec)) + nodes, _, _, _ = check_semantic_cache([str(f)], root=tmp_path, prompt_file=str(spec)) + assert [n["id"] for n in nodes] == ["v1"] + + # An upgrade rewrites the spec: the same file path is now a different prompt. + import os as _os + spec.write_text("prompt two — rewritten by an upgrade", encoding="utf-8") + _os.utime(spec, ns=(0, 0)) # force a distinct stat signature + _, _, _, uncached = check_semantic_cache([str(f)], root=tmp_path, prompt_file=str(spec)) + assert uncached == [str(f)], "an edited spec must invalidate, not reuse the memo" diff --git a/tests/test_cargo_introspect.py b/tests/test_cargo_introspect.py index e80d8c16a..9abf82655 100644 --- a/tests/test_cargo_introspect.py +++ b/tests/test_cargo_introspect.py @@ -363,3 +363,95 @@ def test_cargo_introspect_large_workspace_dependency_chain(tmp_path): "source_file": "crates/crate_198/Cargo.toml", "source_location": "L1", } in result["edges"] + + +def test_cargo_introspect_honors_package_rename_on_internal_dep(tmp_path): + """Renamed workspace-internal deps still produce a `crate_depends_on` edge (#1858). + + Cargo's `package = "..."` inside a dep table entry lets the key used in + `use db::…;` differ from the crate's real `[package].name`. Looking up + `crates` by the raw dep-table key misses the rename and silently drops + the edge. + """ + _write_manifest( + tmp_path / "Cargo.toml", + """ +[workspace] +members = ["app", "storage"] +""", + ) + app = tmp_path / "app" + storage = tmp_path / "storage" + app.mkdir() + storage.mkdir() + _write_manifest( + app / "Cargo.toml", + """ +[package] +name = "app" +version = "0.1.0" +edition = "2021" + +[dependencies] +db = { path = "../storage", package = "internal-storage" } +""", + ) + _write_manifest( + storage / "Cargo.toml", + """ +[package] +name = "internal-storage" +version = "0.1.0" +edition = "2021" +""", + ) + + result = introspect_cargo(tmp_path) + + node_ids = {node["id"] for node in result["nodes"]} + assert node_ids == {"crate:app", "crate:internal-storage"} + assert { + "source": "crate:app", + "target": "crate:internal-storage", + "relation": "crate_depends_on", + "context": "cargo_dependency", + "weight": 1.0, + "confidence": "EXTRACTED", + "source_file": "app/Cargo.toml", + "source_location": "L1", + } in result["edges"] + + +def test_cargo_introspect_package_rename_falls_through_when_unresolved(tmp_path): + """A rename pointing at an external (non-workspace) crate stays a no-op. + + Guards against the fix silently emitting edges to registry crates: the + resolved name still has to appear in `crates` (the workspace-internal + index) for an edge to be produced. + """ + _write_manifest( + tmp_path / "Cargo.toml", + """ +[workspace] +members = ["app"] +""", + ) + app = tmp_path / "app" + app.mkdir() + _write_manifest( + app / "Cargo.toml", + """ +[package] +name = "app" +version = "0.1.0" +edition = "2021" + +[dependencies] +tokio_rt = { version = "1", package = "tokio" } +""", + ) + + result = introspect_cargo(tmp_path) + + assert {node["id"] for node in result["nodes"]} == {"crate:app"} + assert result["edges"] == [] diff --git a/tests/test_chunking.py b/tests/test_chunking.py index 21b28eec4..dea9f68f2 100644 --- a/tests/test_chunking.py +++ b/tests/test_chunking.py @@ -118,7 +118,9 @@ def test_estimate_file_tokens_uses_tiktoken_when_available(tmp_path): # Force the tokenizer to be a mock that records calls and returns a known # token list, so we can assert the tiktoken path is taken. - fake_encoder = type("E", (), {"encode": staticmethod(lambda s: [0] * 999)})() + # Match tiktoken's real signature: encode(text, *, disallowed_special=...) + # so the #1685 hardening call (disallowed_special=()) reaches the mock. + fake_encoder = type("E", (), {"encode": staticmethod(lambda s, **kw: [0] * 999)})() with patch.object(llm, "_TOKENIZER", fake_encoder): n = llm._estimate_file_tokens(f) assert n == 999 + (llm._PER_FILE_OVERHEAD_CHARS // llm._CHARS_PER_TOKEN) @@ -277,6 +279,280 @@ def maybe_fail(chunk, **kwargs): assert "failed" in err and "simulated API error" in err +def test_checkpoint_scopes_cache_writes_to_chunk_files(tmp_path): + """#1757: the per-chunk incremental checkpoint must not let a chunk's + mis-attributed node clobber another corpus file's semantic cache. A chunk + processing only A.py that returns a node attributed to B.py must leave B.py's + existing cache entry untouched. Guards the call site the original fix missed.""" + from graphify.llm import extract_corpus_parallel + from graphify.cache import save_semantic_cache, load_cached + + a = tmp_path / "A.py"; a.write_text("def a(): pass") + b = tmp_path / "B.py"; b.write_text("def b(): pass") + + # Seed B.py's legitimate semantic cache (a full, correct entry). + save_semantic_cache( + [{"id": "b_real", "source_file": "B.py", "file_type": "code"}], + [], [], root=tmp_path, + ) + before = load_cached(b, tmp_path, kind="semantic") + assert before and [n["id"] for n in before["nodes"]] == ["b_real"] + + # The chunk dispatches only A.py, but the (untrusted) model result attributes + # a stray node to B.py — the #1757 mis-attribution. + def stray(chunk, **kwargs): + return { + "nodes": [ + {"id": "a_ok", "source_file": "A.py", "file_type": "code"}, + {"id": "b_stray", "source_file": "B.py", "file_type": "code"}, + ], + "edges": [], "hyperedges": [], + "input_tokens": 1, "output_tokens": 1, + } + + with patch("graphify.llm.extract_files_direct", side_effect=stray): + extract_corpus_parallel( + [a], backend="kimi", root=tmp_path, + token_budget=None, chunk_size=1, max_concurrency=1, + ) + + # B.py's cache is unchanged: the stray node was rejected, not merged in. + after = load_cached(b, tmp_path, kind="semantic") + assert [n["id"] for n in after["nodes"]] == ["b_real"], ( + f"B.py cache was clobbered by an out-of-chunk node: {after}" + ) + # A.py (the actual chunk file) was legitimately cached. The checkpoint stamps + # entries with the prompt that produced them (#1939), so read that namespace. + from graphify.llm import _extraction_system + + a_cache = load_cached(a, tmp_path, kind="semantic", prompt=_extraction_system()) + assert a_cache and any(n["id"] == "a_ok" for n in a_cache["nodes"]) + + +def test_truncated_chunk_is_cached_partial_and_missed_on_reload(tmp_path): + """A single-file chunk that stays truncated is checkpointed as a PARTIAL + entry, so reloading it is a cache miss (the file re-dispatches next run) + instead of serving the incomplete node set forever.""" + from graphify.llm import extract_corpus_parallel, _extraction_system + from graphify.cache import load_cached + + doc = tmp_path / "doc.md"; doc.write_text("# Heading\nlots of prose\n") + + def truncated(chunk, **kwargs): + return { + "nodes": [{"id": "n1", "source_file": "doc.md", "file_type": "document"}], + "edges": [], "hyperedges": [], + "input_tokens": 1, "output_tokens": 1, + "finish_reason": "length", + } + + with patch("graphify.llm.extract_files_direct", side_effect=truncated): + extract_corpus_parallel( + [doc], backend="kimi", root=tmp_path, + token_budget=None, chunk_size=1, max_concurrency=1, + ) + + # The entry was written but stamped partial, so load_cached rejects it. + assert load_cached(doc, tmp_path, kind="semantic", prompt=_extraction_system()) is None + + +def test_checkpoint_writes_deep_namespace_in_deep_mode(tmp_path): + """#1894: the per-chunk checkpoint must follow the run's mode — a + deep_mode=True run checkpoints into cache/semantic-deep/, leaving the + standard cache/semantic/ namespace untouched (and vice versa).""" + from graphify.llm import extract_corpus_parallel, _extraction_system + from graphify.cache import load_cached + + doc = tmp_path / "doc.md" + doc.write_text("# Doc\n\nsome content\n") + + def ok(chunk, **kwargs): + return { + "nodes": [{"id": "d1", "source_file": "doc.md", "file_type": "document"}], + "edges": [], "hyperedges": [], "input_tokens": 1, "output_tokens": 1, + } + + with patch("graphify.llm.extract_files_direct", side_effect=ok): + extract_corpus_parallel( + [doc], backend="kimi", root=tmp_path, + token_budget=None, chunk_size=1, max_concurrency=1, + deep_mode=True, + ) + + # The checkpoint also stamps entries with the prompt that produced them + # (#1939) — a deep run's prompt carries the deep suffix. + deep = load_cached(doc, tmp_path, kind="semantic-deep", + prompt=_extraction_system(deep=True)) + assert deep and [n["id"] for n in deep["nodes"]] == ["d1"], ( + "deep-mode checkpoint must land in cache/semantic-deep/" + ) + assert load_cached(doc, tmp_path, kind="semantic", + prompt=_extraction_system(deep=False)) is None, ( + "deep-mode checkpoint must not write the standard semantic namespace" + ) + + +def test_omitted_documents_are_reconciled_and_warned(tmp_path, capsys): + """#1890: a chunk can return a clean, non-empty response that omits some of the + documents it was given. Those docs must not vanish silently — the run reports + them in `uncovered_files` and warns, instead of dropping them with no signal.""" + from graphify.llm import extract_corpus_parallel + + docs = [] + for i in range(4): + f = tmp_path / f"doc{i}.md" + f.write_text(f"# Doc {i}\n\nsome content\n", encoding="utf-8") + docs.append(f) + + def omit_odd(chunk, **kwargs): + # Return nodes only for even-numbered docs; a clean response, not a failure. + nodes = [] + for u in chunk: + name = getattr(u, "path", u).name + idx = int(name[len("doc")]) + if idx % 2 == 0: + nodes.append({"id": f"n{idx}", "source_file": name, "file_type": "document"}) + return {"nodes": nodes, "edges": [], "hyperedges": [], "input_tokens": 1, "output_tokens": 1} + + with patch("graphify.llm.extract_files_direct", side_effect=omit_odd): + result = extract_corpus_parallel( + docs, backend="kimi", root=tmp_path, + token_budget=None, chunk_size=1, max_concurrency=1, + ) + + uncovered = {Path(p).name for p in result.get("uncovered_files", [])} + assert uncovered == {"doc1.md", "doc3.md"}, f"reconciliation missed omissions: {uncovered}" + err = capsys.readouterr().err + assert "produced no nodes" in err and "doc1.md" in err + + +def test_out_of_scope_nodes_are_dropped_from_merged_result(tmp_path, capsys): + """#1895: the #1757 cache guard skips the CACHE write for a node attributed + to a real corpus file that was not dispatched, but the node itself still + flowed into merged["nodes"] and landed in graph.json. The merged result must + drop such nodes (and edges/hyperedges touching them), warn once, and record + the count — while keeping in-scope sibling attributions (a node attributed + to a different dispatched file in the same chunk) and non-file concept + source_files, mirroring the #1757 `.is_file()` condition.""" + from graphify.llm import extract_corpus_parallel + + a = tmp_path / "A.md"; a.write_text("# a\n") + c = tmp_path / "C.md"; c.write_text("# c\n") + # B.py exists on disk but is NOT dispatched — the #1895 out-of-scope case. + b = tmp_path / "B.py"; b.write_text("def b(): pass\n") + + def stray(chunk, **kwargs): + return { + "nodes": [ + {"id": "a_ok", "source_file": "A.md", "file_type": "document"}, + # sibling attribution: a different dispatched file in the same chunk + {"id": "c_sibling", "source_file": "C.md", "file_type": "document"}, + # out-of-scope: real file on disk, never dispatched + {"id": "b_stray", "source_file": "B.py", "file_type": "code"}, + # concept node: source_file is not a file — must survive + {"id": "auth_flow", "source_file": "auth flow", "file_type": "concept"}, + ], + "edges": [ + {"source": "a_ok", "target": "c_sibling", "source_file": "A.md"}, + {"source": "a_ok", "target": "b_stray", "source_file": "A.md"}, + ], + "hyperedges": [ + {"id": "h_bad", "nodes": ["a_ok", "c_sibling", "b_stray"], "source_file": "A.md"}, + {"id": "h_ok", "nodes": ["a_ok", "c_sibling", "auth_flow"], "source_file": "A.md"}, + ], + "input_tokens": 1, "output_tokens": 1, + } + + with patch("graphify.llm.extract_files_direct", side_effect=stray): + result = extract_corpus_parallel( + [a, c], backend="kimi", root=tmp_path, + token_budget=None, chunk_size=2, max_concurrency=1, + ) + + ids = {n["id"] for n in result["nodes"]} + assert "b_stray" not in ids, "out-of-scope node leaked into the merged graph (#1895)" + assert {"a_ok", "c_sibling", "auth_flow"} <= ids, ( + f"in-scope sibling/concept attributions must be kept: {ids}" + ) + assert result["out_of_scope_dropped"] == 1 + # Edges/hyperedges referencing the dropped node id are gone; in-scope ones stay. + assert all( + "b_stray" not in (e.get("source"), e.get("target")) for e in result["edges"] + ), f"edge to dropped node survived: {result['edges']}" + assert any( + e["source"] == "a_ok" and e["target"] == "c_sibling" for e in result["edges"] + ) + assert [h["id"] for h in result["hyperedges"]] == ["h_ok"] + err = capsys.readouterr().err + assert "out-of-scope" in err and "B.py" in err + # The dispatched files all produced nodes — reconciliation sees no gaps. + assert result["uncovered_files"] == [] + + +def test_out_of_scope_drop_count_is_zero_when_all_in_scope(tmp_path, capsys): + """Counter-test: a clean run records out_of_scope_dropped == 0 and no warning.""" + from graphify.llm import extract_corpus_parallel + + a = tmp_path / "A.md"; a.write_text("# a\n") + + def clean(chunk, **kwargs): + return { + "nodes": [{"id": "a_ok", "source_file": "A.md", "file_type": "document"}], + "edges": [], "hyperedges": [], "input_tokens": 1, "output_tokens": 1, + } + + with patch("graphify.llm.extract_files_direct", side_effect=clean): + result = extract_corpus_parallel( + [a], backend="kimi", root=tmp_path, + token_budget=None, chunk_size=1, max_concurrency=1, + ) + + assert result["out_of_scope_dropped"] == 0 + assert [n["id"] for n in result["nodes"]] == ["a_ok"] + assert "out-of-scope" not in capsys.readouterr().err + + +def test_checkpoint_caches_sliced_document_chunks(tmp_path, capsys): + """#1870: the checkpoint's allowlist must resolve a FileSlice to its parent + path (via unit_path), not read a non-existent `.rel`. An oversized doc is + split into FileSlice units; before the fix each sliced chunk leaked the + FileSlice object into the allowlist, so save_semantic_cache raised TypeError, + the best-effort except swallowed it, and the slice was never checkpointed.""" + from graphify.llm import ( + extract_corpus_parallel, expand_oversized_files, _FILE_CHAR_CAP, _extraction_system, + ) + from graphify.file_slice import FileSlice + from graphify.cache import load_cached + + doc = tmp_path / "big.md" + doc.write_text("# Title\n" + ("word " * 12000) + "\n## Section\n" + ("more " * 12000)) + # sanity: the doc really does slice into FileSlice units + units = expand_oversized_files([doc], _FILE_CHAR_CAP) + assert len(units) > 1 and all(isinstance(u, FileSlice) for u in units) + + def sliced(chunk, **kwargs): + assert any(isinstance(c, FileSlice) for c in chunk) + return { + "nodes": [{"id": "big_title", "source_file": "big.md", "file_type": "document"}], + "edges": [], "hyperedges": [], "input_tokens": 1, "output_tokens": 1, + } + + with patch("graphify.llm.extract_files_direct", side_effect=sliced): + extract_corpus_parallel( + [doc], backend="kimi", root=tmp_path, + token_budget=None, chunk_size=1, max_concurrency=1, + ) + + assert "incremental cache checkpoint failed" not in capsys.readouterr().err, ( + "checkpoint raised on a FileSlice chunk (#1870)" + ) + # The checkpoint stamps entries with the prompt that produced them (#1939). + cached = load_cached(doc, tmp_path, kind="semantic", prompt=_extraction_system()) + assert cached and any(n["id"] == "big_title" for n in cached["nodes"]), ( + "sliced document was never checkpointed (#1870)" + ) + + def test_corpus_parallel_legacy_mode_when_token_budget_is_none(tmp_path): """token_budget=None should fall back to legacy fixed-count chunking.""" from graphify.llm import extract_corpus_parallel @@ -463,6 +739,68 @@ def stub(chunk, **kwargs): assert "single-file chunk" in err and "truncated" in err +def test_adaptive_retry_marks_single_file_truncation_partial(tmp_path): + """A non-splittable single-file truncation keeps its partial result but + marks every item ``_partial`` so it is not cached as complete.""" + from graphify.llm import _extract_with_adaptive_retry + + f = tmp_path / "huge.py"; f.write_text("x") + + def stub(chunk, **kwargs): + return _stub_with_finish(len(chunk), finish_reason="length") + + with patch("graphify.llm.extract_files_direct", side_effect=stub): + result = _extract_with_adaptive_retry( + [f], backend="kimi", api_key=None, model=None, root=tmp_path, max_depth=3 + ) + + assert result["nodes"], "the partial result should still be returned" + assert all(n.get("_partial") for n in result["nodes"]) + + +def test_adaptive_retry_marks_max_depth_giveup_partial(tmp_path): + """When recursion caps at max_depth with everything still truncated, the + merged partial result is marked ``_partial`` on every item.""" + from graphify.llm import _extract_with_adaptive_retry + + files = [tmp_path / f"f{i}.py" for i in range(8)] + for f in files: + f.write_text("x") + + def stub(chunk, **kwargs): + return _stub_with_finish(len(chunk), finish_reason="length") + + with patch("graphify.llm.extract_files_direct", side_effect=stub): + result = _extract_with_adaptive_retry( + files, backend="kimi", api_key=None, model=None, root=tmp_path, max_depth=2 + ) + + assert result["nodes"] + assert all(n.get("_partial") for n in result["nodes"]) + + +def test_adaptive_retry_successful_split_is_not_marked_partial(tmp_path): + """A truncation that IS recovered by splitting yields a complete result — + it must NOT carry the partial marker.""" + from graphify.llm import _extract_with_adaptive_retry + + files = [tmp_path / f"f{i}.py" for i in range(4)] + for f in files: + f.write_text("x") + + def stub(chunk, **kwargs): + finish = "length" if len(chunk) == 4 else "stop" + return _stub_with_finish(len(chunk), finish_reason=finish) + + with patch("graphify.llm.extract_files_direct", side_effect=stub): + result = _extract_with_adaptive_retry( + files, backend="kimi", api_key=None, model=None, root=tmp_path, max_depth=3 + ) + + assert result["nodes"] + assert not any(n.get("_partial") for n in result["nodes"]) + + def test_corpus_parallel_uses_adaptive_retry(tmp_path): """End-to-end: extract_corpus_parallel routes through adaptive retry, so a chunk that truncates gets split and merged transparently before @@ -497,3 +835,30 @@ def stub(chunk, **kwargs): assert len(chunk_done_args) == 1 assert chunk_done_args[0] == (0, 1, 4) assert len(result["nodes"]) == 4 + + +# ---- #1685: special-token strings in docs must not crash token estimation ---- + +def test_estimate_file_tokens_handles_tiktoken_special_token(tmp_path): + """A doc containing a literal tiktoken special token (e.g. <|endoftext|>) + must not crash token estimation. tiktoken's default encode() raises on such + strings appearing as ordinary text; we pass disallowed_special=() since this + is only an estimate (#1685).""" + import graphify.llm as llm + if llm._TOKENIZER is None: + import pytest + pytest.skip("tiktoken not installed; estimation uses the char heuristic") + f = tmp_path / "tokenizer-notes.md" + f.write_text("The GPT end-of-text token is <|endoftext|> in the vocab.\n") + n = llm._estimate_file_tokens(f) # must not raise + assert isinstance(n, int) and n > 0 + + +def test_pack_chunks_with_special_token_doc_does_not_crash(tmp_path): + """End to end: packing a corpus that includes a special-token doc must not + raise (the crash in #1685 happened during token-budget packing).""" + from graphify.llm import _pack_chunks_by_tokens + doc = tmp_path / "doc.md"; doc.write_text("see <|endoftext|> and <|im_start|> tokens\n") + code = tmp_path / "code.py"; code.write_text("def f():\n return 1\n") + chunks = _pack_chunks_by_tokens([doc, code], token_budget=60_000) + assert chunks # produced at least one chunk, no exception diff --git a/tests/test_cjs_module_extension.py b/tests/test_cjs_module_extension.py new file mode 100644 index 000000000..f94f962ff --- /dev/null +++ b/tests/test_cjs_module_extension.py @@ -0,0 +1,90 @@ +"""CommonJS module extension (`.cjs`) is treated as code. + +`.cjs` is the explicit-CommonJS counterpart of `.mjs` (used pervasively in +Electron main/preload scripts and in `"type": "module"` packages that need a +CommonJS escape hatch). The language maps in `build.py` and `extract.py` +already routed `.cjs` to the JS grammar, but the extension was missing from +`CODE_EXTENSIONS`, the extractor `_DISPATCH`, `_LANG_FAMILY`, and the JS +resolution/cache sets — so `.cjs` sources were silently skipped during a build +(detected as non-code and never handed to the JS extractor). These are +regression locks for the extension sets plus an end-to-end extraction proving +`.cjs` parses identically to `.js`. + +Same shape of gap (and fix) as `.mts`/`.cts` in +tests/test_typescript_module_extensions.py. +""" +from __future__ import annotations + +from pathlib import Path + + +def _labels(r): + return [n["label"] for n in r["nodes"]] + + +def test_cjs_registered_as_code(): + from graphify.detect import CODE_EXTENSIONS + assert ".cjs" in CODE_EXTENSIONS + + +def test_cjs_in_extractor_dispatch(): + from graphify.extract import _DISPATCH, extract_js + assert _DISPATCH.get(".cjs") is extract_js + + +def test_cjs_in_js_language_family(): + from graphify.analyze import _LANG_FAMILY + assert _LANG_FAMILY.get(".cjs") == "js" + + +def test_cjs_in_js_resolution_sets(): + from graphify.extract import _JS_CACHE_BYPASS_SUFFIXES, _JS_RESOLVE_EXTS + assert ".cjs" in _JS_RESOLVE_EXTS + assert ".cjs" in _JS_CACHE_BYPASS_SUFFIXES + + +def test_cjs_in_hook_source_exts(): + from graphify.cli import _HOOK_SOURCE_EXTS + assert ".cjs" in _HOOK_SOURCE_EXTS + + +# A representative CommonJS source: require() imports, a class, a function, and +# module.exports — the shape of an Electron main-process script. +_CJS_SOURCE = ( + "const path = require('path');\n" + "const { app, BrowserWindow } = require('electron');\n" + "class WindowManager {\n" + " open() { return new BrowserWindow(); }\n" + "}\n" + "function createWindow() {\n" + " const manager = new WindowManager();\n" + " return manager.open();\n" + "}\n" + "module.exports = { createWindow };\n" +) + + +def _extract(tmp_path: Path, ext: str): + from graphify.extract import extract_js + f = tmp_path / f"main{ext}" + f.write_text(_CJS_SOURCE, encoding="utf-8") + return extract_js(f) + + +def test_cjs_extracts_like_js(tmp_path): + # `.cjs` must parse identically to the same source saved as `.js` — same + # node set (require() imports, class, function) modulo the file-node label. + cjs = _extract(tmp_path, ".cjs") + js = _extract(tmp_path, ".js") + assert "error" not in cjs + cjs_labels = set(_labels(cjs)) + js_labels = set(_labels(js)) + assert any("WindowManager" in label for label in cjs_labels), ( + ".cjs class declaration missing — file was not parsed as JS" + ) + assert any("createWindow" in label for label in cjs_labels), ( + ".cjs function declaration missing — file was not parsed as JS" + ) + assert {label for label in cjs_labels if not label.endswith(".cjs")} == { + label for label in js_labels if not label.endswith(".js") + } diff --git a/tests/test_claude_cli_backend.py b/tests/test_claude_cli_backend.py index c695fb62d..7699ad82e 100644 --- a/tests/test_claude_cli_backend.py +++ b/tests/test_claude_cli_backend.py @@ -154,6 +154,63 @@ def test_user_turn_preserves_untrusted_source_guardrails(fake_claude): assert "untrusted_source" in sent +# ---------- structured output via --json-schema (#2076) ---------- +# Newer Claude Code CLIs treat a bare file-dump prompt as an agentic task and +# REPORT the extraction in prose instead of returning JSON, so the graph comes +# out empty and adaptive-retry bisects forever. When the CLI supports +# `--json-schema`, graphify constrains the output shape structurally so the +# model must emit the object regardless of framing. Older CLIs that predate the +# flag fall back to the user-turn prompt, unchanged. + + +def test_json_schema_flag_added_when_cli_supports_it(monkeypatch, fake_claude): + """When the CLI advertises --json-schema, it is passed with a schema that + pins the top-level {nodes, edges} shape graphify parses.""" + monkeypatch.setattr(llm, "_claude_cli_supports_json_schema", lambda cmd: True) + llm._call_claude_cli("dummy source", max_tokens=8192) + argv = fake_claude.call_args.args[0] + assert "--json-schema" in argv + schema = json.loads(argv[argv.index("--json-schema") + 1]) + assert schema["type"] == "object" + assert set(schema["required"]) == {"nodes", "edges"} + + +def test_json_schema_flag_absent_when_cli_lacks_it(monkeypatch, fake_claude): + """Older CLIs without --json-schema must not receive the flag (it would be + an unknown option) — extraction falls back to the user-turn prompt.""" + monkeypatch.setattr(llm, "_claude_cli_supports_json_schema", lambda cmd: False) + result = llm._call_claude_cli("dummy source", max_tokens=8192) + argv = fake_claude.call_args.args[0] + assert "--json-schema" not in argv + assert len(result["nodes"]) == 2 # result envelope still carries the JSON + + +def test_supports_json_schema_detects_flag_in_help(): + llm._JSON_SCHEMA_SUPPORT.clear() + help_text = "Options:\n --json-schema JSON Schema for structured output\n" + completed = MagicMock(returncode=0, stdout=help_text, stderr="") + with patch("subprocess.run", return_value=completed): + assert llm._claude_cli_supports_json_schema("/fake/claude-new") is True + + +def test_supports_json_schema_false_when_flag_absent(): + llm._JSON_SCHEMA_SUPPORT.clear() + help_text = "Options:\n --output-format text|json|stream-json\n" + completed = MagicMock(returncode=0, stdout=help_text, stderr="") + with patch("subprocess.run", return_value=completed): + assert llm._claude_cli_supports_json_schema("/fake/claude-old") is False + + +def test_supports_json_schema_false_and_cached_on_probe_error(): + """A probe that fails to run is treated as unsupported (safe fallback) and + cached so it is not re-probed for every chunk.""" + llm._JSON_SCHEMA_SUPPORT.clear() + with patch("subprocess.run", side_effect=OSError("boom")) as run: + assert llm._claude_cli_supports_json_schema("/fake/claude-broken") is False + assert llm._claude_cli_supports_json_schema("/fake/claude-broken") is False + assert run.call_count == 1 + + # ---------- Windows path resolution (#1072) ---------- @@ -292,3 +349,35 @@ def fake_which(name): assert out == "ok" assert captured["argv0"] == r"C:\npm\claude.cmd" + + +def test_prefers_structured_output_over_prose_result(monkeypatch): + """#2076 review: with --json-schema the CLI puts the constrained object in + `structured_output` while `result` may be prose (a 'reporting' turn). The + backend must parse the structured object; parsing the prose would read as an + empty/hollow extraction and bisect forever.""" + envelope = { + "type": "result", "subtype": "success", "is_error": False, + "result": "Knowledge graph extracted successfully: 2 nodes, 1 edge.", # prose only + "structured_output": { + "nodes": [ + {"id": "foo_module", "label": "Foo", "file_type": "document", "source_file": "foo.md"}, + {"id": "foo_greet", "label": "greet", "file_type": "code", "source_file": "foo.md"}, + ], + "edges": [ + {"source": "foo_module", "target": "foo_greet", + "relation": "references", "confidence": "EXTRACTED", "confidence_score": 1.0}, + ], + "hyperedges": [], + }, + "stop_reason": "end_turn", + "usage": {"input_tokens": 6, "output_tokens": 11}, + "modelUsage": {"claude-opus-4-7[1m]": {}}, + } + completed = MagicMock(returncode=0, stdout=json.dumps(envelope), stderr="") + with patch("shutil.which", return_value="/fake/bin/claude"), \ + patch("subprocess.run", return_value=completed): + result = llm._call_claude_cli("dummy", max_tokens=8192) + assert len(result["nodes"]) == 2, "must parse structured_output, not the prose result" + assert len(result["edges"]) == 1 + assert result["finish_reason"] == "stop" diff --git a/tests/test_claude_md.py b/tests/test_claude_md.py index f81f10dd3..cac794083 100644 --- a/tests/test_claude_md.py +++ b/tests/test_claude_md.py @@ -109,7 +109,7 @@ def test_install_creates_settings_json(tmp_path): assert settings_path.exists() settings = json.loads(settings_path.read_text()) hooks = settings.get("hooks", {}).get("PreToolUse", []) - assert any(h.get("matcher") == "Bash" for h in hooks) + assert any(h.get("matcher") == "Bash|Grep" for h in hooks) def test_install_settings_json_idempotent(tmp_path): @@ -120,7 +120,7 @@ def test_install_settings_json_idempotent(tmp_path): settings_path = tmp_path / ".claude" / "settings.json" settings = json.loads(settings_path.read_text()) hooks = settings.get("hooks", {}).get("PreToolUse", []) - bash_hooks = [h for h in hooks if h.get("matcher") == "Bash" and "graphify" in str(h)] + bash_hooks = [h for h in hooks if h.get("matcher") == "Bash|Grep" and "graphify" in str(h)] assert len(bash_hooks) == 1 @@ -133,4 +133,73 @@ def test_uninstall_removes_settings_hook(tmp_path): if settings_path.exists(): settings = json.loads(settings_path.read_text()) hooks = settings.get("hooks", {}).get("PreToolUse", []) - assert not any(h.get("matcher") == "Bash" and "graphify" in str(h) for h in hooks) + assert not any(h.get("matcher") == "Bash|Grep" and "graphify" in str(h) for h in hooks) + + +# --------------------------------------------------------------------------- +# local-only variants: settings.local.json / CLAUDE.local.md (#1731) +# --------------------------------------------------------------------------- + +def test_uninstall_removes_hook_from_settings_local_json(tmp_path): + """A hook relocated to .claude/settings.local.json is removed on uninstall.""" + import json + claude_install(tmp_path) + # User moved the hook out of the committed settings.json into the local-only file. + (tmp_path / ".claude" / "settings.json").rename(tmp_path / ".claude" / "settings.local.json") + claude_uninstall(tmp_path) + local = tmp_path / ".claude" / "settings.local.json" + hooks = json.loads(local.read_text()).get("hooks", {}).get("PreToolUse", []) + assert not any("graphify" in str(h) for h in hooks) + + +def test_uninstall_removes_section_from_dot_claude_local_md(tmp_path): + """Instructions relocated to .claude/CLAUDE.local.md are removed on uninstall.""" + claude_install(tmp_path) + local_md = tmp_path / ".claude" / "CLAUDE.local.md" + local_md.write_text((tmp_path / "CLAUDE.md").read_text()) + (tmp_path / "CLAUDE.md").unlink() + claude_uninstall(tmp_path) + assert not local_md.exists() or _CLAUDE_MD_MARKER not in local_md.read_text() + + +def test_uninstall_removes_section_from_root_claude_local_md(tmp_path): + """Instructions relocated to root CLAUDE.local.md are removed on uninstall.""" + claude_install(tmp_path) + local_md = tmp_path / "CLAUDE.local.md" + local_md.write_text((tmp_path / "CLAUDE.md").read_text()) + (tmp_path / "CLAUDE.md").unlink() + claude_uninstall(tmp_path) + assert not local_md.exists() or _CLAUDE_MD_MARKER not in local_md.read_text() + + +def test_uninstall_cleans_both_standard_and_local(tmp_path): + """When the section lives in both CLAUDE.md and a local variant, both are cleaned.""" + claude_install(tmp_path) + claude_md = tmp_path / "CLAUDE.md" + local_md = tmp_path / ".claude" / "CLAUDE.local.md" + local_md.write_text(claude_md.read_text()) # duplicated into the local file too + claude_uninstall(tmp_path) + for f in (claude_md, local_md): + assert not f.exists() or _CLAUDE_MD_MARKER not in f.read_text() + + +def test_uninstall_preserves_other_content_in_local_md(tmp_path): + """Uninstall keeps non-graphify content in CLAUDE.local.md.""" + claude_install(tmp_path) + local_md = tmp_path / ".claude" / "CLAUDE.local.md" + local_md.write_text("# Local notes\n\nkeep me\n\n" + (tmp_path / "CLAUDE.md").read_text()) + claude_uninstall(tmp_path) + assert local_md.exists() + content = local_md.read_text() + assert "Local notes" in content + assert "keep me" in content + assert _CLAUDE_MD_MARKER not in content + + +def test_uninstall_tolerates_unreadable_local_md(tmp_path): + """A non-UTF-8 CLAUDE.local.md must not abort uninstall (it has no marker to strip).""" + claude_install(tmp_path) + local_md = tmp_path / ".claude" / "CLAUDE.local.md" + local_md.write_bytes(b"\xff\xfe not valid utf-8 \x80\x81") + claude_uninstall(tmp_path) # must not raise + assert local_md.read_bytes() == b"\xff\xfe not valid utf-8 \x80\x81" # left untouched diff --git a/tests/test_cli_broken_pipe.py b/tests/test_cli_broken_pipe.py new file mode 100644 index 000000000..3c7da17ce --- /dev/null +++ b/tests/test_cli_broken_pipe.py @@ -0,0 +1,49 @@ +"""CLI must not crash when a downstream reader closes the pipe early (#1807). + +Truncating a command's output (`head`, PowerShell `Select-Object -First N`, +`sed q`) is routine. graphify used to keep writing after the reader disconnected, +hit an unhandled BrokenPipeError, and exit 255 — so CI wrappers and agent +harnesses that both trim output and check the exit code read a successful query +as a failure. An early-closing reader is now treated as success (exit 0). +""" +from __future__ import annotations + +import subprocess +import sys + +PYTHON = sys.executable + + +def test_help_survives_reader_closing_pipe_early(): + """`graphify --help | head -n1` must leave graphify exiting 0, not 255.""" + producer = subprocess.Popen( + [PYTHON, "-m", "graphify", "--help"], stdout=subprocess.PIPE + ) + reader = subprocess.Popen( + [PYTHON, "-c", "import sys; sys.stdin.readline()"], + stdin=producer.stdout, + stdout=subprocess.DEVNULL, + ) + producer.stdout.close() # let the producer see EPIPE when the reader exits + reader.wait() + rc = producer.wait() + # 0 (our handled-and-succeed convention). Never the 255 unhandled-exception code. + assert rc == 0, f"expected clean exit after early pipe close, got {rc}" + + +def test_small_buffered_output_survives_reader_that_reads_nothing(): + """A short, fully-buffered output (piped stdout is block-buffered) only flushes + at exit. If the reader closed the pipe without reading, that flush must be + handled inside the CLI's guard and exit 0, not escape as a shutdown error.""" + producer = subprocess.Popen( + [PYTHON, "-m", "graphify", "--version"], stdout=subprocess.PIPE + ) + reader = subprocess.Popen( + [PYTHON, "-c", "pass"], # exits immediately, reads nothing + stdin=producer.stdout, + stdout=subprocess.DEVNULL, + ) + producer.stdout.close() + reader.wait() + rc = producer.wait() + assert rc == 0, f"expected clean exit when reader reads nothing, got {rc}" diff --git a/tests/test_cli_export.py b/tests/test_cli_export.py index 879cb68b5..6173a61aa 100644 --- a/tests/test_cli_export.py +++ b/tests/test_cli_export.py @@ -321,6 +321,42 @@ def test_cluster_only_creates_output_dir_when_missing(tmp_path): assert (tmp_path / "graphify-out" / "GRAPH_REPORT.md").exists() +def test_cluster_only_graph_in_graphify_out_writes_beside_it(tmp_path): + """#1747 Case 2: `cluster-only --graph /graphify-out/graph.json` + must write GRAPH_REPORT.md and the re-clustered graph beside that graph, not + into a stray graphify-out/ in the CWD.""" + project = tmp_path / "project" + project.mkdir() + out_dir = _make_graph(project) # project/graphify-out/graph.json + + cwd = tmp_path / "elsewhere" + cwd.mkdir() + r = _run( + ["cluster-only", ".", "--graph", str(out_dir / "graph.json"), "--no-viz", "--no-label"], + cwd, + ) + assert r.returncode == 0, r.stderr + assert (out_dir / "GRAPH_REPORT.md").exists() # beside --graph + assert not (cwd / "graphify-out").exists() # no CWD pollution + + +def test_extract_out_does_not_pollute_corpus(tmp_path): + """#1747 Case 1: `extract --out ` must not leave a stray + graphify-out/ (cache, stat-index) inside the scanned corpus.""" + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "a.py").write_text("def main():\n return 1\n") + out = tmp_path / "scratch" + + r = _run( + ["extract", str(corpus), "--out", str(out), "--no-cluster", "--code-only"], + tmp_path, + ) + assert r.returncode == 0, r.stderr + assert (out / "graphify-out" / "graph.json").exists() # graph in --out + assert not (corpus / "graphify-out").exists() # corpus untouched + + # Regression test for #1027 - cluster-only must remap labels via node overlap def test_cluster_only_persists_analysis_sidecar(tmp_path): @@ -484,3 +520,24 @@ def test_export_html_no_community_data_at_all_still_succeeds(tmp_path): # code stays clean — same behaviour as the pre-fallback empty-communities # path, just no longer silently failing on the common case. assert r.returncode == 0, r.stderr + + +def test_graph_json_node_ids_are_portable_across_checkout_paths(tmp_path): + """#1789: the committed graph.json's node ids must be relative to the scan + root — not embed the absolute path — so the same repo yields identical ids + on any machine/checkout and leaks no local username/home.""" + def _build(root: Path): + (root / "pkg").mkdir(parents=True) + (root / "pkg" / "mod.py").write_text("def f(): return 1\n") + (root / "pkg" / "app.py").write_text("from pkg.mod import f\ndef g(): return f()\n") + r = _run(["extract", ".", "--code-only", "--no-cluster"], root) + assert r.returncode == 0, r.stderr + data = json.loads((root / "graphify-out" / "graph.json").read_text()) + return sorted(n["id"] for n in data["nodes"]) + + a = _build(tmp_path / "alice_home" / "proj") + b = _build(tmp_path / "bob_elsewhere" / "checkout" / "proj") + assert a == b, f"node ids differ across checkout paths: {a} vs {b}" + leak = {"alice_home", "bob_elsewhere", "checkout", "tmp", "private", "users", "home", "var"} + assert not any(part in leak for ident in a for part in ident.split("_")), \ + f"node id embeds an absolute-path component: {a}" diff --git a/tests/test_codebuddy.py b/tests/test_codebuddy.py index a4850105f..62a7e6c02 100644 --- a/tests/test_codebuddy.py +++ b/tests/test_codebuddy.py @@ -98,7 +98,7 @@ def test_codebuddy_install_hook_has_bash_matcher(tmp_path): codebuddy_install(tmp_path) settings = json.loads(_settings_path(tmp_path).read_text()) hooks = settings["hooks"]["PreToolUse"] - bash_hooks = [h for h in hooks if h.get("matcher") == "Bash"] + bash_hooks = [h for h in hooks if h.get("matcher") == "Bash|Grep"] assert any("graphify" in str(h) for h in bash_hooks) diff --git a/tests/test_cross_extension_reexport_self_cycle.py b/tests/test_cross_extension_reexport_self_cycle.py new file mode 100644 index 000000000..7b66699ac --- /dev/null +++ b/tests/test_cross_extension_reexport_self_cycle.py @@ -0,0 +1,311 @@ +"""Same-basename cross-extension re-exports must not collapse to a self-cycle (#1814). + +A hand-written ``.mjs`` plain-ESM runtime plus a thin typed ``.ts`` wrapper that +re-exports it (``export { N } from "./foo.mjs"``) is a common convention. Because +``_file_stem`` drops the extension, ``foo.ts`` and ``foo.mjs`` both collapse to the +base file id ``foo`` at extract time. ``_disambiguate_colliding_node_ids`` salts the +two NODES apart correctly (``foo_ts_foo`` / ``foo_mjs_foo``), but the file-level +re-export edge (and the ``re_exports`` export edge) keyed its target salt by the +IMPORTER's own source_file, mis-pointing the ``./foo.mjs`` target back onto the +importer's own variant — a phantom ``foo.ts -> foo.ts`` self-loop reported as a +1-file import cycle. + +These lock: the salted node ids stay unchanged (the fix does NOT make ids +extension-aware — Option A rejected, #1033), the re-export edge lands on the +sibling node, and no phantom 1-file cycle survives. +""" +from __future__ import annotations + +import json +from pathlib import Path + +from graphify.build import build +from graphify.export import to_json +from graphify.extract import extract + + +def _write(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def _node_id_by_label(result: dict, label: str) -> str: + ids = [n["id"] for n in result["nodes"] if n.get("label") == label] + assert len(ids) == 1, f"expected exactly one node labelled {label!r}; got {ids}" + return ids[0] + + +def _reexport_like_edges(result: dict) -> list[dict]: + return [ + e for e in result["edges"] + if e.get("relation") in ("imports_from", "re_exports") + ] + + +def test_cross_ext_reexport_emits_no_self_loop(tmp_path: Path): + mjs = _write(tmp_path / "foo.mjs", "export const N = 1;\n") + ts = _write(tmp_path / "foo.ts", 'export { N } from "./foo.mjs";\n') + + result = extract([mjs, ts], cache_root=tmp_path) + + self_loops = [ + e for e in _reexport_like_edges(result) + if e.get("source") == e.get("target") + ] + assert not self_loops, ( + f"cross-extension re-export produced a phantom self-loop; got {self_loops}" + ) + + +def test_cross_ext_reexport_target_is_the_sibling_node(tmp_path: Path): + mjs = _write(tmp_path / "foo.mjs", "export const N = 1;\n") + ts = _write(tmp_path / "foo.ts", 'export { N } from "./foo.mjs";\n') + + result = extract([mjs, ts], cache_root=tmp_path) + + foo_ts = _node_id_by_label(result, "foo.ts") + foo_mjs = _node_id_by_label(result, "foo.mjs") + # The scheme stays as-is: extension dropped from the base stem, siblings salted + # apart by source path. The fix must NOT make node ids extension-aware (#1814 + # Option A rejected). + assert foo_ts == "foo_ts_foo" + assert foo_mjs == "foo_mjs_foo" + + file_level = [ + e for e in result["edges"] + if e.get("relation") == "imports_from" and e.get("source") == foo_ts + ] + assert file_level, "no file-level re-export edge from foo.ts was emitted" + assert all(e.get("target") == foo_mjs for e in file_level), ( + f"file-level re-export must target the .mjs sibling node {foo_mjs!r}; " + f"got {[e.get('target') for e in file_level]}" + ) + + # The symbol-provenance re_exports (context='export') edge must also point at + # the sibling, never back at the importer. + export_edges = [ + e for e in result["edges"] + if e.get("relation") == "re_exports" and e.get("context") == "export" + and e.get("source") == foo_ts + ] + assert export_edges, "no re_exports export edge from foo.ts was emitted" + assert all(e.get("target") == foo_mjs for e in export_edges), ( + f"re_exports export edge must target the .mjs sibling node {foo_mjs!r}; " + f"got {[e.get('target') for e in export_edges]}" + ) + + +def test_cross_ext_reexport_no_phantom_import_cycle(tmp_path: Path): + import networkx as nx + + from graphify.analyze import find_import_cycles + + mjs = _write(tmp_path / "foo.mjs", "export const N = 1;\n") + ts = _write(tmp_path / "foo.ts", 'export { N } from "./foo.mjs";\n') + + result = extract([mjs, ts], cache_root=tmp_path) + + graph = nx.DiGraph() + for node in result["nodes"]: + graph.add_node(node["id"], **{k: v for k, v in node.items() if k != "id"}) + for edge in result["edges"]: + graph.add_edge( + edge["source"], + edge["target"], + **{k: v for k, v in edge.items() if k not in ("source", "target")}, + ) + assert find_import_cycles(graph) == [], ( + "cross-extension re-export must not manufacture a file-level import cycle" + ) + + +def test_same_basename_three_colliding_siblings_reexport_selects_named_variant( + tmp_path: Path, +): + """With three same-basename siblings that all collapse to the base id ``foo`` + (``foo.mjs`` / ``foo.cjs`` / ``foo.ts``), keying the re-export target by the + RESOLVED target file — not the importer's file — must land on the specifically + named ``./foo.mjs`` variant, proving the fix is a real per-file selection and + not a binary coin-flip between two colliders. + + (The issue's illustrative trio uses ``foo.d.mts``, but a ``.d.mts`` stem keeps + its ``.d`` segment and so does NOT collide with ``foo`` — it cannot exercise + multi-variant salt selection. ``foo.cjs`` is the realistic dual-format sibling + that genuinely collides.) + """ + mjs = _write(tmp_path / "foo.mjs", "export const N = 1;\n") + cjs = _write(tmp_path / "foo.cjs", "module.exports.M = 2;\n") + ts = _write(tmp_path / "foo.ts", 'export { N } from "./foo.mjs";\n') + + result = extract([mjs, cjs, ts], cache_root=tmp_path) + + foo_ts = _node_id_by_label(result, "foo.ts") + foo_mjs = _node_id_by_label(result, "foo.mjs") + foo_cjs = _node_id_by_label(result, "foo.cjs") + assert foo_mjs != foo_cjs != foo_ts + + file_level = [ + e for e in result["edges"] + if e.get("relation") == "imports_from" and e.get("source") == foo_ts + ] + assert file_level, "no file-level re-export edge from foo.ts was emitted" + assert all(e.get("target") == foo_mjs for e in file_level), ( + f"re-export of './foo.mjs' must resolve to the .mjs node {foo_mjs!r}, not " + f"the .cjs sibling {foo_cjs!r}; got {[e.get('target') for e in file_level]}" + ) + self_loops = [ + e for e in _reexport_like_edges(result) + if e.get("source") == e.get("target") + ] + assert not self_loops, f"unexpected self-loop among siblings; got {self_loops}" + + +# --------------------------------------------------------------------------- # +# The ``target_file`` the fix stamps on import/re-export edges is a transient +# extraction-time disambiguation salt hint (its only reader is the salt lookup +# in ``_disambiguate_colliding_node_ids``). It carries an ABSOLUTE filesystem +# path, so it must never survive its consumer onto a persisted edge: leaking it +# into graph.json breaks determinism across checkout locations and the +# cross-machine merge/global-graph portability the codebase engineered for. The +# following lock that the hint is stripped after disambiguation and never +# reaches graph.json — on the raw-dump extract path AND the build path — and +# that a persisted absolute hint from a pre-fix graph is dropped on the next +# build rather than carried forward. +# --------------------------------------------------------------------------- # + + +def test_disambiguation_strips_transient_target_file_hint(tmp_path: Path): + mjs = _write(tmp_path / "foo.mjs", "export const N = 1;\n") + ts = _write(tmp_path / "foo.ts", 'export { N } from "./foo.mjs";\n') + + result = extract([mjs, ts], cache_root=tmp_path) + + leaked = [e for e in result["edges"] if "target_file" in e] + assert not leaked, ( + f"the target_file salt hint must not survive disambiguation onto an " + f"edge (it carries an absolute path with no downstream reader); got {leaked}" + ) + + +def test_target_file_hint_stripped_even_without_a_collision(tmp_path: Path): + # No same-basename collision here, so `_disambiguate_colliding_node_ids` + # takes its early `if not remap: return` exit before the edge loop. An + # ordinary import still stamps target_file at extraction, so that early + # exit must strip it too — otherwise every non-colliding import leaks an + # absolute path. + util = _write(tmp_path / "util.ts", "export const helper = 1;\n") + main = _write(tmp_path / "main.ts", 'import { helper } from "./util";\n') + + result = extract([util, main], cache_root=tmp_path) + + leaked = [e for e in result["edges"] if "target_file" in e] + assert not leaked, ( + f"a non-colliding import leaked the transient target_file hint; got {leaked}" + ) + + +def test_graph_json_has_no_target_file_and_no_absolute_path(tmp_path: Path): + mjs = _write(tmp_path / "foo.mjs", "export const N = 1;\n") + ts = _write(tmp_path / "foo.ts", 'export { N } from "./foo.mjs";\n') + + result = extract([mjs, ts], cache_root=tmp_path) + graph = build([result], root=tmp_path) + out = tmp_path / "graph.json" + to_json(graph, {}, str(out), force=True) + + raw = out.read_text(encoding="utf-8") + data = json.loads(raw) + leaked = [link for link in data["links"] if "target_file" in link] + assert not leaked, f"absolute target_file persisted into graph.json links: {leaked}" + assert str(tmp_path.resolve()) not in raw, ( + "graph.json leaked an absolute checkout path (source_file is relativized, " + "but the target_file hint was serialized verbatim)" + ) + + +def test_graph_json_is_checkout_location_independent(tmp_path: Path): + """Building the byte-identical repo at two different absolute locations must + yield identical graph.json edges. A leaked absolute target_file differs by + its checkout prefix and would defeat the cross-machine merge/global-graph + portability the codebase is built around.""" + + def _links_built_at(dirname: str) -> list[dict]: + d = tmp_path / dirname + d.mkdir() + mjs = _write(d / "foo.mjs", "export const N = 1;\n") + ts = _write(d / "foo.ts", 'export { N } from "./foo.mjs";\n') + result = extract([mjs, ts], cache_root=d) + graph = build([result], root=d) + out = d / "graph.json" + to_json(graph, {}, str(out), force=True) + links = json.loads(out.read_text(encoding="utf-8"))["links"] + return sorted( + links, + key=lambda link: ( + str(link.get("source")), + str(link.get("target")), + str(link.get("relation")), + ), + ) + + assert _links_built_at("loc_a") == _links_built_at("loc_bbbb_longer"), ( + "graph.json edges differ across checkout locations — an absolute path leaked" + ) + + +def test_build_drops_persisted_target_file_from_a_pre_fix_graph(tmp_path: Path): + # A graph.json written by a pre-fix build carries an absolute target_file on + # its import edges. On the next (incremental) build those base edges are + # re-serialized through build(), which does NOT re-run disambiguation — so + # the serializer itself must drop the persisted absolute path rather than + # carry a foreign checkout prefix forward into the updated graph. + legacy_chunk = { + "nodes": [ + {"id": "foo_ts_foo", "label": "foo.ts", + "source_file": "foo.ts", "file_type": "code"}, + {"id": "foo_mjs_foo", "label": "foo.mjs", + "source_file": "foo.mjs", "file_type": "code"}, + ], + "edges": [ + { + "source": "foo_ts_foo", + "target": "foo_mjs_foo", + "relation": "imports_from", + "context": "re-export", + "confidence": "EXTRACTED", + "source_file": "foo.ts", + "target_file": "/some/other/checkout/foo.mjs", + "weight": 1.0, + } + ], + } + + graph = build([legacy_chunk], root=tmp_path) + + assert graph.number_of_edges() == 1, "the base import edge should survive the merge" + for _src, _tgt, data in graph.edges(data=True): + assert "target_file" not in data, ( + f"build() carried a persisted absolute target_file into the graph: {data}" + ) + + +def test_target_file_hint_never_written_to_the_ast_cache(tmp_path: Path): + """The hint is emitted only on JS/TS-family edges, and those suffixes bypass + the AST cache entirely (``_JS_CACHE_BYPASS_SUFFIXES``). A warm/relocated + cache therefore can never carry a foreign absolute target_file that would + miss the disambiguation salt. Lock that no AST cache entry stores it.""" + mjs = _write(tmp_path / "foo.mjs", "export const N = 1;\n") + ts = _write(tmp_path / "foo.ts", 'export { N } from "./foo.mjs";\n') + + extract([mjs, ts], cache_root=tmp_path) + + ast_dir = tmp_path / "graphify-out" / "cache" / "ast" + entries = list(ast_dir.rglob("*.json")) if ast_dir.exists() else [] + for entry in entries: + payload = json.loads(entry.read_text(encoding="utf-8")) + for edge in payload.get("edges", []): + assert "target_file" not in edge, ( + f"AST cache entry {entry.name} stored a non-portable target_file " + f"hint: {edge}" + ) diff --git a/tests/test_cross_language_call_resolution.py b/tests/test_cross_language_call_resolution.py new file mode 100644 index 000000000..62fe4c60f --- /dev/null +++ b/tests/test_cross_language_call_resolution.py @@ -0,0 +1,103 @@ +"""Cross-language call resolution — a call in one language must never bind by +name to a definition in another language family. + +The cross-file resolver matched raw-call callees against a repo-wide label +index with no language check, so in a repo that mixes a web app with a native +Android app a TSX callback passed by name (``register(refreshHeading)``) +resolved to a same-named Kotlin method and shipped as an INFERRED +``indirect_call`` edge — a phantom the extraction spec explicitly forbids +("calls edges MUST stay within one language"). Direct calls from non-JS/TS +languages had the same hole: a Python call bound to a Kotlin ``fun``. + +The fix filters resolution candidates by language interop family. Families are +grouped by REAL interop so legitimate cross-language resolution keeps working: +Kotlin/Java share the JVM, C/C++/Objective-C share headers, JS/TS variants +compile into one module graph. Candidates with no known family (non-code +nodes) are never filtered, preserving the previous permissive behavior. +""" +from __future__ import annotations + +from pathlib import Path + +from graphify.extract import extract + + +def _write(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def _call_edges(files: list[Path], base: Path) -> set[tuple[str, str, str, str]]: + r = extract(files, cache_root=base, parallel=False) + lbl = {n["id"]: n["label"] for n in r["nodes"]} + return { + (lbl.get(e["source"], ""), lbl.get(e["target"], ""), e["relation"], e.get("confidence")) + for e in r["edges"] if e["relation"] in ("calls", "indirect_call") + } + + +def test_tsx_callback_does_not_bind_to_kotlin_method(tmp_path: Path) -> None: + # The real-world symptom: a repo with a web app and a native Android app. + # A TSX component passes a callback by name; the only same-named definition + # repo-wide is a Kotlin method. No edge must be emitted. + _write(tmp_path / "web/Upcoming.tsx", + "declare function register(cb: () => void): void;\n" + "export function UpcomingPanel() {\n" + " register(refreshHeading);\n" + " return null;\n" + "}\n") + _write(tmp_path / "android/HeadingSensorBridge.kt", + "class HeadingSensorBridge {\n" + " fun refreshHeading() {\n" + " println(\"native sensor\")\n" + " }\n" + "}\n") + edges = _call_edges(sorted(tmp_path.rglob("*.tsx")) + sorted(tmp_path.rglob("*.kt")), tmp_path) + assert not any("refreshHeading" in t for _s, t, _r, _c in edges), edges + + +def test_python_call_does_not_bind_to_kotlin_function(tmp_path: Path) -> None: + # Direct-call path (non-JS/TS callers have no import-evidence gate): a bare + # Python call must not resolve to the lone same-named Kotlin definition. + _write(tmp_path / "py/worker.py", + "def process():\n" + " return refreshHeading()\n") + _write(tmp_path / "android/HeadingSensorBridge.kt", + "class HeadingSensorBridge {\n" + " fun refreshHeading() {\n" + " println(\"native sensor\")\n" + " }\n" + "}\n") + edges = _call_edges(sorted(tmp_path.rglob("*.py")) + sorted(tmp_path.rglob("*.kt")), tmp_path) + assert not any("refreshHeading" in t for _s, t, _r, _c in edges), edges + + +def test_same_language_callback_still_resolves(tmp_path: Path) -> None: + # Positive control: a TS callback passed by name with a same-language + # definition and import evidence keeps resolving as INFERRED indirect_call. + _write(tmp_path / "a.ts", + 'import { refreshHeading } from "./b";\n' + "declare function register(cb: () => void): void;\n" + "export function run() { register(refreshHeading); }\n") + _write(tmp_path / "b.ts", + "export function refreshHeading(): void {}\n") + edges = _call_edges([tmp_path / "a.ts", tmp_path / "b.ts"], tmp_path) + resolved = [e for e in edges if "refreshHeading" in e[1] and e[2] == "indirect_call"] + assert resolved, edges + assert resolved[0][3] == "INFERRED" + + +def test_jvm_interop_kotlin_call_to_java_still_resolves(tmp_path: Path) -> None: + # Kotlin and Java share the JVM — same interop family, so a Kotlin call to a + # Java method must keep resolving exactly as it did before the guard. + _write(tmp_path / "Alarm.java", + "public class Alarm {\n" + " public static void ring() { System.out.println(\"ring\"); }\n" + "}\n") + _write(tmp_path / "Scheduler.kt", + "fun schedule() {\n" + " ring()\n" + "}\n") + edges = _call_edges([tmp_path / "Alarm.java", tmp_path / "Scheduler.kt"], tmp_path) + assert any("ring" in t for _s, t, _r, _c in edges), edges diff --git a/tests/test_csharp_member_calls.py b/tests/test_csharp_member_calls.py index d83265d9b..5487939c1 100644 --- a/tests/test_csharp_member_calls.py +++ b/tests/test_csharp_member_calls.py @@ -141,3 +141,211 @@ def test_unqualified_call_still_resolves(tmp_path): ) }) assert any("_r_a" in s and "helper" in t for s, t in calls), "no regression on unqualified calls" + + +# ── Namespace-aware receiver typing + shadow poisoning (#1620) ──────────────── + +_NS_AB = { + "A.cs": "namespace A { public class Svc { public bool Do() => true; } }\n", + "B.cs": "namespace B { public class Svc { public bool Do() => false; } }\n", +} + + +def test_namespace_using_directive_disambiguates_receiver_type(tmp_path): + """`Svc` exists in namespaces A and B; a caller file `using A;` must bind an + `A.Svc`-typed receiver to A.Svc.Do — before #1620 the corpus-wide bare-name + ambiguity made the resolver bail (missing edge).""" + calls, r = _calls(tmp_path, { + **_NS_AB, + "Caller.cs": ( + "using A;\n" + "namespace App {\n" + " public class Runner { public bool Go(Svc s) { return s.Do(); } }\n" + "}\n" + ), + }) + a_do = _find(r, ".Do()", "a_a_svc") + b_do = _find(r, ".Do()", "b_b_svc") + runner_go = _find(r, ".Go()", "runner") + assert (runner_go, a_do) in calls, "using A; must resolve Svc to A.Svc" + assert (runner_go, b_do) not in calls, "must NOT bind to the same-named B.Svc" + + +def test_namespace_using_directive_resolves_to_other_namespace(tmp_path): + calls, r = _calls(tmp_path, { + **_NS_AB, + "Caller.cs": ( + "using B;\n" + "namespace App {\n" + " public class Runner { public bool Go(Svc s) { return s.Do(); } }\n" + "}\n" + ), + }) + a_do = _find(r, ".Do()", "a_a_svc") + b_do = _find(r, ".Do()", "b_b_svc") + runner_go = _find(r, ".Go()", "runner") + assert (runner_go, b_do) in calls, "using B; must resolve Svc to B.Svc" + assert (runner_go, a_do) not in calls + + +def test_namespace_ambiguous_without_using_bails(tmp_path): + """No using directive and `Svc` in two foreign namespaces: genuinely + ambiguous — no edge to either candidate (never a guess).""" + calls, r = _calls(tmp_path, { + **_NS_AB, + "Caller.cs": ( + "namespace App {\n" + " public class Runner { public bool Go(Svc s) { return s.Do(); } }\n" + "}\n" + ), + }) + assert not any("runner" in s and "svc_do" in t for s, t in calls), \ + "ambiguous cross-namespace type must produce no edge" + + +def test_same_namespace_receiver_resolves_without_using(tmp_path): + """A caller in namespace A resolves `Svc` to A.Svc even though B.Svc also + exists — same-namespace visibility needs no using directive.""" + calls, r = _calls(tmp_path, { + **_NS_AB, + "A2.cs": ( + "namespace A {\n" + " public class Client { public bool Go(Svc s) { return s.Do(); } }\n" + "}\n" + ), + }) + a_do = _find(r, ".Do()", "a_a_svc") + b_do = _find(r, ".Do()", "b_b_svc") + client_go = _find(r, ".Go()", "client") + assert (client_go, a_do) in calls + assert (client_go, b_do) not in calls + + +def test_local_shadowing_field_of_different_type_poisons_name(tmp_path): + """A local `Other x` shadowing a field `Server x` makes the name's type + conflicting — the binding is poisoned and `x.Run()` emits NO edge, instead + of first-binding-wins mis-binding to Server.Run (a wrong edge).""" + calls, r = _calls(tmp_path, { + "S.cs": ( + "public class Server { public bool Run() => true; }\n" + "public class Other { public bool Run() => false; }\n" + "public class Holder {\n" + " private Server x = new Server();\n" + " public bool A() { Other x = new Other(); return x.Run(); }\n" + "}\n" + ) + }) + holder_a = _find(r, ".A()", "holder") + server_run = _find(r, ".Run()", "server") + other_run = _find(r, ".Run()", "other") + assert (holder_a, server_run) not in calls, \ + "shadowed field's type must not win (wrong edge)" + assert (holder_a, other_run) not in calls, \ + "conflicting bindings poison the name entirely (conservative: no edge)" + + +def test_untyped_redeclaration_poisons_typed_field(tmp_path): + """`var x = Compute();` (untypable) redeclaring a typed field poisons the + name: `x.Run()` must not bind to the field's type.""" + calls, r = _calls(tmp_path, { + "S.cs": ( + "public class Server { public bool Run() => true; }\n" + "public class Holder {\n" + " private Server x = new Server();\n" + " public object Compute() => new object();\n" + " public bool A() { var x = Compute(); return x.Run(); }\n" + "}\n" + ) + }) + assert not any("holder_a" in s and "run" in t.lower() for s, t in calls) + + +def test_this_field_receiver_resolves(tmp_path): + """`this._s.Save()` types the field exactly like a bare `_s.Save()`.""" + calls, r = _calls(tmp_path, { + "S.cs": ( + "public class Server { public bool Save() => true; }\n" + "public class Cache { public bool Save() => false; }\n" + "public class Repo {\n" + " private Server _s = new Server();\n" + " public bool Commit() { return this._s.Save(); }\n" + "}\n" + ) + }) + commit = _find(r, ".Commit()", "commit") + server_save = _find(r, ".Save()", "server") + cache_save = _find(r, ".Save()", "cache") + assert (commit, server_save) in calls + assert (commit, cache_save) not in calls + + +def test_base_receiver_resolves_to_base_class_method(tmp_path): + calls, r = _calls(tmp_path, { + "Base.cs": "public class BaseSvc { public bool Ping() => true; }\n", + "Sub.cs": ( + "public class Sub : BaseSvc {\n" + " public bool Go() { return base.Ping(); }\n" + "}\n" + ), + }) + sub_go = _find(r, ".Go()", "sub") + ping = _find(r, ".Ping()", "basesvc") + assert (sub_go, ping) in calls, "base.Ping() must resolve to the base class method" + + +def test_inherited_method_resolves_through_base_chain(tmp_path): + """A method not declared on the receiver's type but inherited from a + resolvable in-corpus base resolves to the base's declaration.""" + calls, r = _calls(tmp_path, { + "S.cs": ( + "public class BaseSvc { public bool Ping() => true; }\n" + "public class Derived : BaseSvc { }\n" + "public class User {\n" + " public bool Use(Derived d) { return d.Ping(); }\n" + "}\n" + ) + }) + use = _find(r, ".Use()", "user") + ping = _find(r, ".Ping()", "basesvc") + assert (use, ping) in calls + + +def test_unresolved_base_poisons_inherited_member_lookup(tmp_path): + """The receiver's type inherits from an out-of-corpus base: a method missing + on the type may live on that base, so the lookup is poisoned — and it must + NOT fall back to an unrelated same-named in-corpus method.""" + calls, r = _calls(tmp_path, { + "S.cs": ( + "public class Server { public bool Save() => true; }\n" + "public class Ext : NotInCorpus { }\n" + "public class User {\n" + " public bool U(Ext e) { return e.Save(); }\n" + "}\n" + ) + }) + assert not any("user_u" in s and "save" in t.lower() for s, t in calls), \ + "unresolved base chain must bail, not mis-bind to Server.Save" + + +def test_method_chained_off_new_expression_resolves(tmp_path): + """#1770: a method invoked directly on a `new X(...)` object-creation + expression (no intermediate variable) must still emit a calls edge to the + constructed type's method — the fluent `new X(...).M()` pattern.""" + calls, r = _calls(tmp_path, { + "S.cs": ( + "public class Merger {\n" + " public Merger(int x) {}\n" + " public int Combine(int a, bool b) { return a; }\n" + "}\n" + "public class Svc {\n" + " public int Run(int ctx) {\n" + " return new Merger(ctx).Combine(ctx, true);\n" + " }\n" + "}\n" + ) + }) + label = {n["id"]: n.get("label") for n in r["nodes"]} + assert any( + "run" in s and label.get(t) == ".Combine()" + for s, t in calls + ), f"chained call off new Merger(...) not captured: {[(s, label.get(t)) for s, t in calls]}" diff --git a/tests/test_dedup.py b/tests/test_dedup.py index 267388e57..e1370fc7a 100644 --- a/tests/test_dedup.py +++ b/tests/test_dedup.py @@ -1,7 +1,7 @@ """Tests for graphify/dedup.py entity deduplication pipeline.""" from __future__ import annotations import pytest -from graphify.dedup import deduplicate_entities, _entropy, _shingles +from graphify.dedup import deduplicate_entities, _defines_id, _entropy, _shingles # ── entropy gate ───────────────────────────────────────────────────────────── @@ -137,6 +137,30 @@ def test_build_calls_dedup(): assert G.number_of_nodes() == 1 +def test_build_dedup_preserves_semantic_attributes(): + """The default build path must not discard semantic enrichment (#2091).""" + from graphify.build import build + ast = { + "nodes": [{"id": "src_auth_login", "label": "login", "file_type": "code", + "source_file": "src/auth.py", "_origin": "ast", + "source_location": "L42"}], + "edges": [], + } + semantic = { + "nodes": [{"id": "src_auth_login", "label": "User login handler", + "file_type": "code", "source_file": "src/auth.py", + "summary": "Authenticates a user.", "confidence_score": 0.9}], + "edges": [], + } + + node = dict(build([ast, semantic], directed=True, dedup=True).nodes["src_auth_login"]) + + assert node["label"] == "login" + assert node["source_location"] == "L42" + assert node["summary"] == "Authenticates a user." + assert node["confidence_score"] == 0.9 + + # --- #878: fuzzy dedup false merges on short/variant labels --- def test_dedup_does_not_merge_numeric_variants(tmp_path): @@ -404,3 +428,444 @@ def test_same_id_same_source_file_no_warning(capsys): assert len(result_nodes) == 1 captured = capsys.readouterr() assert "WARNING" not in captured.err + + +# ── #1857: dedup summary log breakdown ──────────────────────────────────────── + +def test_dedup_summary_prints_fuzzy_count_when_no_exact_merges(capsys): + """A fuzzy-only run must still report the fuzzy count (#1857). + + Two long, high-entropy, non-code labels on different files. Pass 1 (exact, + same-file) finds nothing; Pass 2 (Jaro-Winkler cross-file) merges them. + Previously the fuzzy count was nested inside `if exact_merges`, so this + line printed as `Deduplicated 1 node(s).` with no breakdown. + """ + nodes = [ + {"id": "g1", "label": "GraphExtractor", "file_type": "concept", "source_file": "a.md"}, + {"id": "g2", "label": "Graph Extractor", "file_type": "concept", "source_file": "b.md"}, + ] + result_nodes, _ = deduplicate_entities(nodes, [], communities={}) + assert len(result_nodes) == 1 + captured = capsys.readouterr() + assert "Deduplicated" in captured.out + assert "fuzzy" in captured.out + assert "exact" not in captured.out + + +def test_dedup_summary_still_reports_exact_only(capsys): + """Non-regression: an exact-only run still prints `(N exact)` and no fuzzy.""" + # Same file + same normalized label → Pass 1 exact merge. + nodes = [ + {"id": "u1", "label": "User Service", "file_type": "concept", "source_file": "svc.md"}, + {"id": "u2", "label": "user service", "file_type": "concept", "source_file": "svc.md"}, + ] + result_nodes, _ = deduplicate_entities(nodes, [], communities={}) + assert len(result_nodes) == 1 + captured = capsys.readouterr() + assert "exact" in captured.out + assert "fuzzy" not in captured.out + + +# ── ID collisions: definition vs cross-reference ────────────────────────────── + +# The defining node and a doc that merely mentions the entity. Both mint the ID +# encoded from the *defining* file's path, so they collide by construction. +_DEFINING = {"id": "agents_make_batch_fixtures_make_batch_fixtures", + "label": "make-batch-fixtures agent", "file_type": "concept", + "source_file": "agents/make-batch-fixtures.md"} +_REFERENCING = {"id": "agents_make_batch_fixtures_make_batch_fixtures", + "label": "make-batch-fixtures", "file_type": "concept", + "source_file": "available/diagnose-issue/SKILL.md"} + + +@pytest.mark.parametrize("nodes", [ + [_DEFINING, _REFERENCING], + [_REFERENCING, _DEFINING], +], ids=["definition-first", "reference-first"]) +def test_defining_file_wins_over_referencing_file(nodes, capsys): + """The node whose source_file is the file its ID encodes survives, whichever + chunk order the nodes arrive in — the survivor must not depend on it.""" + result_nodes, _ = deduplicate_entities(list(nodes), [], communities={}) + + assert len(result_nodes) == 1 + assert result_nodes[0]["source_file"] == "agents/make-batch-fixtures.md" + assert result_nodes[0]["label"] == "make-batch-fixtures agent" + + +def test_reference_collision_is_silent(capsys): + """A cross-reference rewires silently without importing foreign-file metadata.""" + edges = _make_edges("agents_make_batch_fixtures_make_batch_fixtures", "other") + referencing = dict(_REFERENCING, summary="Reference-local description.") + result_nodes, result_edges = deduplicate_entities( + [_DEFINING, referencing], edges, communities={}) + + assert len(result_nodes) == 1 + assert len(result_edges) == 1 + assert "summary" not in result_nodes[0] + captured = capsys.readouterr() + assert "WARNING" not in captured.err + assert "note:" not in captured.err + + +def test_absolute_source_path_still_defines_id(capsys): + """source_file is absolute in some pipelines and repo-relative in others; the + defining file is recognised either way.""" + absolute = dict(_DEFINING, source_file="/home/u/proj/agents/make-batch-fixtures.md") + result_nodes, _ = deduplicate_entities([_REFERENCING, absolute], [], communities={}) + + assert len(result_nodes) == 1 + assert result_nodes[0]["label"] == "make-batch-fixtures agent" + assert "WARNING" not in capsys.readouterr().err + + +def test_same_file_relabel_is_noted(capsys): + """Two labels for one ID from one file: the loser's label is discarded, which is + the one drop that used to be silent. It is a note, not a collision warning.""" + nodes = [ + {"id": "agents_make_batch_fixtures_make_batch_fixtures", + "label": "make-batch-fixtures agent", "file_type": "concept", + "source_file": "agents/make-batch-fixtures.md"}, + {"id": "agents_make_batch_fixtures_make_batch_fixtures", + "label": "make-batch-fixtures helper agent", "file_type": "concept", + "source_file": "agents/make-batch-fixtures.md"}, + ] + result_nodes, _ = deduplicate_entities(nodes, [], communities={}) + + assert len(result_nodes) == 1 + captured = capsys.readouterr() + assert "note:" in captured.err + assert "make-batch-fixtures helper agent" in captured.err + assert "WARNING" not in captured.err + + +@pytest.mark.parametrize("nodes", [ + [ + {"id": "src_auth_login", "label": "login", "file_type": "code", + "source_file": "src/auth.py", "_origin": "ast", "source_location": "L42"}, + {"id": "src_auth_login", "label": "User login handler", "file_type": "code", + "source_file": "src/auth.py", "summary": "Authenticates a user.", + "confidence_score": 0.9}, + ], + [ + {"id": "src_auth_login", "label": "User login handler", "file_type": "code", + "source_file": "src/auth.py", "summary": "Authenticates a user.", + "confidence_score": 0.9}, + {"id": "src_auth_login", "label": "login", "file_type": "code", + "source_file": "src/auth.py", "_origin": "ast", "source_location": "L42"}, + ], +], ids=["ast-first", "semantic-first"]) +def test_same_id_same_entity_retains_complementary_attributes(nodes): + """Exact-ID dedup combines AST precision with semantic enrichment (#2091).""" + result_nodes, _ = deduplicate_entities(nodes, [], communities={}) + + assert len(result_nodes) == 1 + assert result_nodes[0] == { + "id": "src_auth_login", + "label": "login", + "file_type": "code", + "source_file": "src/auth.py", + "_origin": "ast", + "source_location": "L42", + "summary": "Authenticates a user.", + "confidence_score": 0.9, + } + + +def test_cross_file_id_collision_does_not_mix_attributes(capsys): + """Two files that both mint one ID remain isolated despite exact-ID dedup.""" + nodes = [ + {"id": "pkg_service_run", "label": "run", "file_type": "code", + "source_file": "pkg/service.py", "source_location": "L10"}, + {"id": "pkg_service_run", "label": "run helper", "file_type": "code", + "source_file": "pkg_service.py", "summary": "Different function."}, + ] + + result_nodes, _ = deduplicate_entities(nodes, [], communities={}) + + assert len(result_nodes) == 1 + assert result_nodes[0]["source_file"] == "pkg/service.py" + assert "summary" not in result_nodes[0] + assert "WARNING" in capsys.readouterr().err + + +def test_collision_survivor_is_order_independent(): + """#1851: definer + same-file relabel + cross-file reference. Across every + insertion order the SAME node (source_file AND label) must survive — the + definer heuristic alone left the label order-dependent among co-definers.""" + import itertools + nid = "agents_make_batch_fixtures_make_batch_fixtures" + definer = {"id": nid, "label": "make-batch-fixtures agent", + "file_type": "concept", "source_file": "agents/make-batch-fixtures.md"} + relabel = {"id": nid, "label": "make-batch-fixtures helper agent", + "file_type": "concept", "source_file": "agents/make-batch-fixtures.md"} + xref = {"id": nid, "label": "make-batch-fixtures", "file_type": "concept", + "source_file": "available/diagnose-issue/SKILL.md"} + survivors = set() + for perm in itertools.permutations([definer, relabel, xref]): + out, _ = deduplicate_entities([dict(n) for n in perm], [], communities={}) + assert len(out) == 1 + survivors.add((out[0]["source_file"], out[0]["label"])) + assert survivors == {("agents/make-batch-fixtures.md", "make-batch-fixtures agent")}, ( + f"non-deterministic collision survivor: {survivors}" + ) + + +def test_bare_file_node_defines_its_own_id(): + """A file-level semantic node whose id is exactly the slugified path (no + `_entity` suffix) must be recognised as defining its id (#1851 tweak).""" + assert _defines_id({"id": "agents_make_batch_fixtures", + "source_file": "agents/make-batch-fixtures.md"}) + + +def test_defines_id_helper(): + assert _defines_id(_DEFINING) + assert not _defines_id(_REFERENCING) + # Pre-#1504 IDs keyed off the bare filename stem. + assert _defines_id({"id": "readme_booking_service", + "source_file": "module-a/README.md"}) + # A path that is merely a string-prefix of the ID's path does not define it. + assert not _defines_id({"id": "agents_foo", "source_file": "agent/foo.md"}) + assert not _defines_id({"id": "docs_intro_foo", "source_file": ""}) + + +# ── #2091 review: attribute-merge correctness (fixes A-D) ───────────────────── + +def test_dedup_gapfill_is_order_independent_with_multiple_losers(): + """(fix A) With 3+ same-ID same-source records, the merged attributes must not + depend on arrival order — the best loser by collision rank supplies each + missing key deterministically, preserving the #1851 order-independence.""" + import itertools + base = [ + {"id": "f", "label": "f", "file_type": "code", "source_file": "m.py", + "source_location": "L1"}, # shortest label -> survivor + {"id": "f", "label": "f helper beta", "file_type": "code", + "source_file": "m.py", "summary": "BETA"}, + {"id": "f", "label": "f helper alpha", "file_type": "code", + "source_file": "m.py", "summary": "ALPHA"}, + ] + seen = set() + for perm in itertools.permutations(base): + nodes, _ = deduplicate_entities([dict(n) for n in perm], [], communities={}) + assert len(nodes) == 1 + seen.add(nodes[0].get("summary")) + assert len(seen) == 1, f"merged summary depends on arrival order: {seen}" + + +def test_dedup_no_attribute_merge_when_source_file_missing(): + """(fix B) Two provenance-less records sharing an ID must NOT cross-pollinate + attributes — '' == '' is not proof of the same symbol (#1178).""" + nodes = [ + {"id": "c", "label": "c", "file_type": "concept", "summary": "A"}, + {"id": "c", "label": "c", "file_type": "concept", "notes": "B"}, + ] + result, _ = deduplicate_entities([dict(n) for n in nodes], [], communities={}) + assert len(result) == 1 + surv = result[0] + assert not ("summary" in surv and "notes" in surv), ( + "provenance-less same-id records must not merge attributes" + ) + + +def test_dedup_survivor_does_not_inherit_false_origin_ast(): + """(fix C) An LLM survivor must not inherit _origin='ast' from a dropped + same-source AST record — a false authority tag is read by ghost-merge/watch.""" + nodes = [ + {"id": "x", "label": "run", "file_type": "code", "source_file": "m.py", + "source_location": "L9"}, # shorter label -> survivor (LLM) + {"id": "x", "label": "run() [ast]", "file_type": "code", "source_file": "m.py", + "source_location": "L2", "_origin": "ast"}, # loser carries _origin=ast + ] + result, _ = deduplicate_entities([dict(n) for n in nodes], [], communities={}) + assert len(result) == 1 + assert result[0].get("_origin") != "ast", "survivor must not inherit a false _origin=ast" + + +def test_dedup_fills_explicit_none_attribute(): + """(fix D) An explicit source_location=None on the survivor is treated as + absent and filled from a same-source record that has a real line (#2091).""" + nodes = [ + {"id": "y", "label": "y", "file_type": "code", "source_file": "m.py", + "source_location": None}, # survivor, explicit None + {"id": "y", "label": "y helper", "file_type": "code", "source_file": "m.py", + "source_location": "L7"}, + ] + result, _ = deduplicate_entities([dict(n) for n in nodes], [], communities={}) + assert len(result) == 1 + assert result[0].get("source_location") == "L7", "explicit-None must be filled from the loser" + + +# ── #2182: cross-file exact-duplicate concepts must merge ───────────────────── + +def test_crossfile_identical_concepts_merge_and_rewire(): + """Two `concept` nodes whose labels are byte-identical after _norm() but + live in different files must merge (#2182). Pass 1 used to defer them to + Pass 2, whose norm-unique candidate filter (`seen_norms`) structurally + cannot form an equal-norm pair — so exact cross-file duplicates were the + one class of duplicate that never merged, while a one-char-different + fuzzy pair did.""" + nodes = [ + {"id": "sz_intl", "label": "SHENZHEN INTERNATIONAL", + "file_type": "concept", "source_file": "doc1.md"}, + {"id": "shenzhen_international_holdings", "label": "Shenzhen international", + "file_type": "concept", "source_file": "doc2.md"}, + {"id": "port_ops", "label": "Port Operations", + "file_type": "concept", "source_file": "doc2.md"}, + ] + edges = [{"source": "shenzhen_international_holdings", "target": "port_ops", + "relation": "operates"}] + result_nodes, result_edges = deduplicate_entities(nodes, edges, communities={}) + ids = {n["id"] for n in result_nodes} + assert len(result_nodes) == 2 + # _pick_winner prefers the shorter, non-chunk-suffixed id. + assert "sz_intl" in ids + assert "shenzhen_international_holdings" not in ids + # The loser's edge is rewired to the winner. + assert result_edges == [ + {"source": "sz_intl", "target": "port_ops", "relation": "operates"}] + + +def test_crossfile_one_char_typo_concepts_still_merge(): + """Non-regression: the near-identical (one-char-different) cross-file pair + that already merged via Pass 2 fuzzy matching must keep merging (#2182).""" + nodes = [ + {"id": "g1", "label": "Authentication Manager", + "file_type": "concept", "source_file": "a.md"}, + {"id": "g2", "label": "Authentication Managr", + "file_type": "concept", "source_file": "b.md"}, + ] + result_nodes, _ = deduplicate_entities(nodes, [], communities={}) + assert len(result_nodes) == 1 + + +_RATIONALE_BOILER = ("Django app config for apps.platform.cards. No business " + "logic here. Domain services live in services.py.") + + +@pytest.mark.parametrize("a,b", [ + ({"id": "d1", "label": "Getting Started Installation Guide", + "file_type": "document", "source_file": "docs/a.md"}, + {"id": "d2", "label": "Getting Started Installation Guide", + "file_type": "document", "source_file": "docs/b.md"}), + ({"id": "r1", "label": _RATIONALE_BOILER, + "file_type": "rationale", "source_file": "apps/platform/cards/apps.py"}, + {"id": "r2", "label": _RATIONALE_BOILER, + "file_type": "rationale", "source_file": "apps/platform/cores/apps.py"}), + ({"id": "backend_a_render_frame", "label": "render_frame", + "file_type": "code", "source_file": "backend_a.py"}, + {"id": "backend_b_render_frame", "label": "render_frame", + "file_type": "code", "source_file": "backend_b.py"}), + ({"id": "web_logo", "label": "logo.png", + "file_type": "image", "source_file": "web/assets/logo.png"}, + {"id": "docs_logo", "label": "logo.png", + "file_type": "image", "source_file": "docs/img/logo.png"}), + ({"id": "logo_concept", "label": "logo.png", + "file_type": "concept", "source_file": "doc1.md"}, + {"id": "logo_image", "label": "logo.png", + "file_type": "image", "source_file": "assets/logo.png"}), + ({"id": "shenzhen_a", "label": "Shenzhen International", + "file_type": "concept", "source_file": ""}, + {"id": "shenzhen_b", "label": "Shenzhen International", + "file_type": "concept", "source_file": ""}), + ({"id": "api_a", "label": "API", + "file_type": "concept", "source_file": "doc1.md"}, + {"id": "api_b", "label": "API", + "file_type": "concept", "source_file": "doc2.md"}), +], ids=["document", "rationale", "code", "image-basename", "concept-image-mixed", + "empty-source-file", "low-entropy-concept"]) +def test_crossfile_identical_labels_stay_distinct_for_guarded_types(a, b): + """The #2182 fix is gated to high-entropy `concept` nodes with provenance + on BOTH sides. Identical labels must NOT merge for: file-anchored types + (document/rationale, #1284), code (#1205), images sharing a basename in + different dirs, mixed concept+image pairs, provenance-less nodes (#1178), + and low-entropy generic labels.""" + result_nodes, _ = deduplicate_entities([dict(a), dict(b)], [], communities={}) + assert len(result_nodes) == 2, ( + f"guarded pair ({a['id']}, {b['id']}) was merged — #2182 fix leaked " + f"past its concept-only gate" + ) + + +def test_cross_repo_guard_still_raises(): + """The cross-repo guard is untouched by #2182: identical concepts from + different repos must still raise, never merge.""" + nodes = [ + {"id": "c1", "label": "Shenzhen International", "file_type": "concept", + "source_file": "doc1.md", "repo": "repo-a"}, + {"id": "c2", "label": "Shenzhen International", "file_type": "concept", + "source_file": "doc2.md", "repo": "repo-b"}, + ] + with pytest.raises(ValueError, match="multiple repos"): + deduplicate_entities(nodes, [], communities={}) + + +def test_crossfile_concept_merge_is_order_independent(): + """Three identical-norm concepts across three files: every input order must + yield the same single survivor (#2182). Winner ids differ in length so + _pick_winner has a unique minimum.""" + import itertools + base = [ + {"id": "shenzhen", "label": "SHENZHEN INTERNATIONAL", + "file_type": "concept", "source_file": "doc1.md"}, + {"id": "shenzhen_intl", "label": "Shenzhen international", + "file_type": "concept", "source_file": "doc2.md"}, + {"id": "shenzhen_international", "label": "shenzhen-international", + "file_type": "concept", "source_file": "doc3.md"}, + ] + survivors = set() + for perm in itertools.permutations(base): + out, _ = deduplicate_entities([dict(n) for n in perm], [], communities={}) + assert len(out) == 1 + survivors.add(out[0]["id"]) + assert survivors == {"shenzhen"}, f"non-deterministic survivor: {survivors}" + + +def test_crossfile_concept_merge_deterministic_across_hash_seeds(): + """#2182 determinism, #1753/#2074 precedent: the survivor must not depend on + PYTHONHASHSEED. pytest fixes the seed per process, so run out-of-process + with shuffled input.""" + import os + import subprocess + import sys + script = ( + "import random, sys\n" + "from graphify.dedup import deduplicate_entities\n" + "nodes = [\n" + " {'id': 'shenzhen', 'label': 'SHENZHEN INTERNATIONAL',\n" + " 'file_type': 'concept', 'source_file': 'doc1.md'},\n" + " {'id': 'shenzhen_intl', 'label': 'Shenzhen international',\n" + " 'file_type': 'concept', 'source_file': 'doc2.md'},\n" + " {'id': 'shenzhen_international', 'label': 'shenzhen-international',\n" + " 'file_type': 'concept', 'source_file': 'doc3.md'},\n" + "]\n" + "random.Random(int(sys.argv[1])).shuffle(nodes)\n" + "out, _ = deduplicate_entities(nodes, [], communities={})\n" + "print(len(out), sorted(n['id'] for n in out)[0])\n" + ) + results = set() + for seed in ("0", "1", "2", "3"): + env = {**os.environ, "PYTHONHASHSEED": seed} + r = subprocess.run( + [sys.executable, "-c", script, seed], + capture_output=True, text=True, env=env, + ) + assert r.returncode == 0, r.stderr + results.add(r.stdout.strip().splitlines()[-1]) + assert results == {"1 shenzhen"}, ( + f"non-deterministic dedup across hash seeds: {results}" + ) + + +def test_crossfile_concept_merge_is_transitive(): + """Exact cross-file matches and a punctuation variant collapse to one + survivor: {'Acme Corp' doc1, 'Acme Corp' doc2, 'Acme Corp.' doc3} all + normalize to 'acme corp' and must transitively union (#2182).""" + nodes = [ + {"id": "acme_corp_one", "label": "Acme Corp", + "file_type": "concept", "source_file": "doc1.md"}, + {"id": "acme_corp_two", "label": "Acme Corp", + "file_type": "concept", "source_file": "doc2.md"}, + {"id": "acme_corp_three", "label": "Acme Corp.", + "file_type": "concept", "source_file": "doc3.md"}, + ] + result_nodes, _ = deduplicate_entities(nodes, [], communities={}) + assert len(result_nodes) == 1 diff --git a/tests/test_detect.py b/tests/test_detect.py index 458bb6a84..15fb570e4 100644 --- a/tests/test_detect.py +++ b/tests/test_detect.py @@ -1,4 +1,6 @@ +import os import unicodedata +import pytest from pathlib import Path from graphify.detect import classify_file, count_words, detect, detect_incremental, save_manifest, FileType, _looks_like_paper, _is_ignored, _load_graphifyignore, _is_sensitive from graphify import detect as detect_mod @@ -22,6 +24,10 @@ def test_classify_powershell_manifest(): def test_classify_markdown(): assert classify_file(Path("README.md")) == FileType.DOCUMENT +def test_classify_skill(): + # #1901: .skill agent files (Markdown with YAML frontmatter) were dropped as unclassified. + assert classify_file(Path("10_Orchestrator.skill")) == FileType.DOCUMENT + def test_classify_pdf(): assert classify_file(Path("paper.pdf")) == FileType.PAPER @@ -130,6 +136,92 @@ def test_graphifyignore_comments_ignored(tmp_path): assert any("other.py" in f for f in result["files"]["code"]) +def test_graphifyignore_utf8_bom_first_pattern_honored(tmp_path): + """A UTF-8 BOM at the start of .graphifyignore must not corrupt the first + pattern (#2163): git strips a single leading BOM, so `*.log` on line 1 + must still exclude app.log.""" + (tmp_path / ".graphifyignore").write_bytes(b"\xef\xbb\xbf*.log\nbuild/\n") + build = tmp_path / "build" + build.mkdir() + (build / "lib.py").write_text("x = 1") + (tmp_path / "app.log").write_text("log line") + (tmp_path / "main.py").write_text("print('hi')") + + result = detect(tmp_path) + all_files = [f for files in result["files"].values() for f in files] + assert not any("app.log" in f for f in all_files), "BOM'd first pattern was dropped" + assert not any("build" in f for f in all_files) + assert any("main.py" in f for f in all_files) + assert result["graphifyignore_patterns"] == 2 + + +def test_gitignore_utf8_bom_matches_git(tmp_path): + """A BOM'd .gitignore first pattern must match, exactly like git (#2163).""" + (tmp_path / ".gitignore").write_bytes(b"\xef\xbb\xbf*.log\n") + (tmp_path / "app.log").write_text("log line") + (tmp_path / "main.py").write_text("print('hi')") + + result = detect(tmp_path) + all_files = [f for files in result["files"].values() for f in files] + assert not any("app.log" in f for f in all_files) + assert any("main.py" in f for f in all_files) + + +def test_graphifyignore_bom_only_file(tmp_path): + """A .graphifyignore containing only a BOM yields zero patterns, not one + bogus U+FEFF pattern (#2163).""" + (tmp_path / ".graphifyignore").write_bytes(b"\xef\xbb\xbf") + (tmp_path / "main.py").write_text("x = 1") + + result = detect(tmp_path) + assert result["graphifyignore_patterns"] == 0 + assert any("main.py" in f for f in result["files"]["code"]) + + +def test_graphifyignore_bom_then_comment(tmp_path): + """A BOM followed by a comment must still parse as a comment, not become + a `\\ufeff# comment` pattern (#2163).""" + (tmp_path / ".graphifyignore").write_bytes(b"\xef\xbb\xbf# comment\nmain.py\n") + (tmp_path / "main.py").write_text("x = 1") + (tmp_path / "other.py").write_text("x = 2") + + result = detect(tmp_path) + assert not any("main.py" in f for f in result["files"]["code"]) + assert any("other.py" in f for f in result["files"]["code"]) + assert result["graphifyignore_patterns"] == 1, "BOM'd comment became a pattern" + + +def test_nested_gitignore_utf8_bom(tmp_path): + """A BOM'd .gitignore below the scan root (loaded live during the walk, + #1206 path) must also have its first pattern honored (#2163).""" + sub = tmp_path / "sub" + sub.mkdir() + (sub / ".gitignore").write_bytes(b"\xef\xbb\xbf*.log\n") + (sub / "app.log").write_text("log line") + (sub / "keep.py").write_text("x = 1") + + result = detect(tmp_path) + all_files = [f for files in result["files"].values() for f in files] + assert not any("app.log" in f for f in all_files) + assert any("keep.py" in f for f in all_files) + + +def test_git_info_exclude_utf8_bom(tmp_path): + """A BOM at the start of $GIT_DIR/info/exclude must not corrupt the first + pattern either (#2163) — second read site in _load_graphifyignore.""" + (tmp_path / ".git" / "info").mkdir(parents=True) + (tmp_path / ".git" / "info" / "exclude").write_bytes(b"\xef\xbb\xbfsecrets/\n") + secrets = tmp_path / "secrets" + secrets.mkdir() + (secrets / "x.py").write_text("token = 'x'") + (tmp_path / "real.py").write_text("def real(): pass") + + result = detect(tmp_path) + all_files = [f for files in result["files"].values() for f in files] + assert not any("secrets" in f for f in all_files), "BOM'd info/exclude pattern was dropped" + assert any("real.py" in f for f in all_files) + + def test_detect_follows_symlinked_directory(tmp_path): real_dir = tmp_path / "real_lib" real_dir.mkdir() @@ -226,6 +318,86 @@ def test_graphifyignore_at_git_root_is_included(tmp_path): assert result["graphifyignore_patterns"] == 1 +def test_gitignore_nested_below_root_excludes_file(tmp_path): + """A .gitignore in a subdirectory below the scan root is honored too (#1206). + + Previously only the scan root and its ancestors were read, so a + .gitignore sitting inside e.g. vendor/sub/ was silently skipped. + """ + (tmp_path / ".gitignore").write_text("*.log\n") + sub = tmp_path / "vendor" / "sub" + sub.mkdir(parents=True) + (sub / ".gitignore").write_text("secret.txt\n") + (tmp_path / "root.py").write_text("x = 1") + (tmp_path / "root.log").write_text("noise") + (sub / "keep.py").write_text("y = 2") + (sub / "secret.txt").write_text("shh") + + result = detect(tmp_path) + code_files = result["files"]["code"] + assert any("root.py" in f for f in code_files) + assert any("keep.py" in f for f in code_files) + assert not any("root.log" in f for f in code_files) + assert not any("secret.txt" in f for f in code_files) + assert result["graphifyignore_patterns"] == 2 + + +def test_gitignore_nested_below_root_prunes_whole_directory(tmp_path): + """A nested .gitignore excluding a directory prevents descending into it.""" + sub = tmp_path / "vendor" / "sub" + sub.mkdir(parents=True) + (sub / ".gitignore").write_text("build/\n") + build = sub / "build" + build.mkdir() + (build / "generated.py").write_text("x = 1") + (sub / "keep.py").write_text("y = 2") + + result = detect(tmp_path) + code_files = result["files"]["code"] + assert any("keep.py" in f for f in code_files) + assert not any("generated.py" in f for f in code_files) + + +def test_gitignore_nested_negation_overrides_broader_root_rule(tmp_path): + """A closer (nested) .gitignore's `!` re-include wins over a root exclude, + matching git's closer-file-wins precedence. Uses .py so classification lands + in the deterministic `code` bucket.""" + (tmp_path / ".gitignore").write_text("*.py\n") + sub = tmp_path / "vendor" / "sub" + sub.mkdir(parents=True) + (sub / ".gitignore").write_text("!important.py\n") + (tmp_path / "root.py").write_text("a = 1") + (sub / "important.py").write_text("b = 1") + (sub / "other.py").write_text("c = 1") + + result = detect(tmp_path) + code = result["files"]["code"] + # nested `!important.py` re-includes it despite the root `*.py` exclude... + assert any("vendor/sub/important.py" in f for f in code) + # ...while the root-excluded and non-re-included files stay out + assert not any(f.endswith("root.py") for f in code) + assert not any(f.endswith("other.py") for f in code) + + +def test_nested_ignore_overrides_git_info_exclude_and_root(tmp_path): + """Precedence across all three sources: a nested `.gitignore` `!` re-include + outranks both a root `.gitignore` and `.git/info/exclude` (lowest, from + #1810), while an info/exclude-only file with no re-include stays out.""" + (tmp_path / ".git" / "info").mkdir(parents=True) + (tmp_path / ".git" / "info" / "exclude").write_text("*.py\n") + (tmp_path / ".gitignore").write_text("keep.py\n") # root also excludes it + sub = tmp_path / "a" / "b" + sub.mkdir(parents=True) + (sub / ".gitignore").write_text("!keep.py\n") # nearest wins -> re-included + (sub / "keep.py").write_text("x = 1") + (tmp_path / "drop.py").write_text("y = 1") # only info/exclude -> excluded + + result = detect(tmp_path) + code = result["files"]["code"] + assert any("a/b/keep.py" in f for f in code), "nested ! must beat root + info/exclude" + assert not any(f.endswith("drop.py") for f in code) + + def test_detect_handles_circular_symlinks(tmp_path): sub = tmp_path / "a" sub.mkdir() @@ -368,6 +540,67 @@ def test_detect_incremental_survives_dict_valued_mtime(tmp_path, monkeypatch): assert not any("mod.py" in f for f in result["unchanged_files"]["code"]) +def test_detect_incremental_legacy_float_reextracts_on_backwards_mtime(tmp_path, monkeypatch): + """Legacy float manifests must re-extract when mtime moves BACKWARDS (#1859). + + Pre-fix the legacy branch used `current_mtime > stored`, which silently kept + the cached entry after operations that restore older mtimes: `git checkout` + of an older commit, `tar -xf` restore, or `rsync --times`. The graph then + reflected the newer content while disk held the older content. The dict + branch has always used `!=`; this test pins the legacy branch to the same + contract. + """ + import json + + monkeypatch.chdir(tmp_path) + + src = tmp_path / "mod.py" + src.write_text("def old_content():\n return 1\n", encoding="utf-8") + current_mtime = os.stat(src).st_mtime + + manifest_dir = tmp_path / "graphify-out" + manifest_dir.mkdir() + manifest_path = str(manifest_dir / "manifest.json") + + # Legacy schema (pre-dict-migration): the value is a bare float mtime. + # Store a mtime FROM THE FUTURE, simulating a checkout of an older + # revision that restored the file to an earlier timestamp. + future_mtime = current_mtime + 3600 + legacy = {str(src.resolve()): future_mtime} + Path(manifest_path).write_text(json.dumps(legacy), encoding="utf-8") + + result = detect_incremental(tmp_path, manifest_path) + + assert any("mod.py" in f for f in result["new_files"]["code"]), ( + "backwards-moving mtime on a legacy manifest entry must trigger re-extract" + ) + assert not any("mod.py" in f for f in result["unchanged_files"]["code"]) + + +def test_detect_incremental_legacy_float_skips_when_mtime_matches(tmp_path, monkeypatch): + """Non-regression for the fix above: legacy float branch still skips when + the stored mtime equals the current mtime.""" + import json + + monkeypatch.chdir(tmp_path) + + src = tmp_path / "mod.py" + src.write_text("def stable():\n return 1\n", encoding="utf-8") + + manifest_dir = tmp_path / "graphify-out" + manifest_dir.mkdir() + manifest_path = str(manifest_dir / "manifest.json") + + # Legacy schema with the exact current mtime → no change → skip. + legacy = {str(src.resolve()): os.stat(src).st_mtime} + Path(manifest_path).write_text(json.dumps(legacy), encoding="utf-8") + + result = detect_incremental(tmp_path, manifest_path) + + assert not any("mod.py" in f for f in result["new_files"]["code"]) + assert any("mod.py" in f for f in result["unchanged_files"]["code"]) + + def test_classify_video_extensions(): """Video and audio file extensions should classify as VIDEO.""" from graphify.detect import FileType @@ -398,7 +631,7 @@ def test_detect_converts_google_workspace_shortcuts_when_enabled(tmp_path, monke shortcut = tmp_path / "notes.gdoc" shortcut.write_text('{"doc_id":"doc-1"}', encoding="utf-8") - def fake_convert(path, out_dir, *, xlsx_to_markdown=None): + def fake_convert(path, out_dir, *, xlsx_to_markdown=None, root=None): out_dir.mkdir(parents=True, exist_ok=True) out = out_dir / "notes_converted.md" out.write_text("# Notes\n\nA converted Google Doc.", encoding="utf-8") @@ -467,16 +700,35 @@ def test_detect_skips_visual_tests_dir(tmp_path): def test_detect_skips_snapshots_dir(tmp_path): - """__snapshots__/ and snapshots/ are jest/vitest artefacts — must be excluded.""" + """__snapshots__/ and real jest/vitest snapshots/ dirs are artefacts — excluded.""" (tmp_path / "__snapshots__").mkdir() (tmp_path / "__snapshots__" / "app.test.ts.snap").write_text("// Jest Snapshot\nexports[`test 1`] = `
`") + # a bare snapshots/ dir that actually holds .snap files is still a JS artefact + snap = tmp_path / "snapshots" + snap.mkdir() + (snap / "component.test.tsx.snap").write_text("exports[`renders`] = ``") (tmp_path / "app.ts").write_text("export function greet() { return 'hi'; }") result = detect(tmp_path) all_files = [f for files in result["files"].values() for f in files] assert not any("__snapshots__" in f for f in all_files) + assert not any(f"{os.sep}snapshots{os.sep}" in f for f in all_files) assert any("app.ts" in f for f in all_files) +def test_detect_keeps_snapshots_code_namespace(tmp_path): + """#1666: a bare snapshots/ dir with no .snap files is a legit code namespace + (e.g. Rails app/services/snapshots/) and must NOT be pruned as a JS artefact.""" + svc = tmp_path / "app" / "services" / "snapshots" + svc.mkdir(parents=True) + (svc / "round_reader.rb").write_text("class RoundReader\n def call; end\nend\n") + (svc / "backfill_marker.rb").write_text("class BackfillMarker\n def run; end\nend\n") + (tmp_path / "app.rb").write_text("class App; end\n") + result = detect(tmp_path) + all_files = [f for files in result["files"].values() for f in files] + assert any("round_reader.rb" in f for f in all_files) + assert any("backfill_marker.rb" in f for f in all_files) + + def test_detect_skips_storybook_static_dir(tmp_path): """storybook-static/ is a build artefact — must be excluded.""" sb = tmp_path / "storybook-static" @@ -517,6 +769,48 @@ def test_detect_skips_next_cache(tmp_path): assert any("index.tsx" in f for f in all_files) +def test_detect_skips_nox_virtualenv(tmp_path): + """.nox/ (nox virtualenvs, tox's successor) must be excluded like .tox (#1804).""" + nox = tmp_path / ".nox" / "tests" / "lib" / "site-packages" / "pydeck" + nox.mkdir(parents=True) + (nox / "widget.py").write_text("class Deck: pass") + (tmp_path / "app.py").write_text("def go(): pass") + result = detect(tmp_path) + all_files = [f for files in result["files"].values() for f in files] + assert not any(".nox" in f for f in all_files) + assert any("app.py" in f for f in all_files) + + +def test_detect_honors_git_info_exclude(tmp_path): + """.git/info/exclude (where `git worktree add` records nested worktree paths, + and where local-only excludes live) must be honored, not just .gitignore / + .graphifyignore — otherwise nested worktree copies get fully indexed (#1810).""" + (tmp_path / ".git" / "info").mkdir(parents=True) + (tmp_path / ".git" / "info" / "exclude").write_text("worktrees/\n") + wt = tmp_path / "worktrees" / "foo" + wt.mkdir(parents=True) + (wt / "dupe.py").write_text("def dupe(): pass") + (tmp_path / "real.py").write_text("def real(): pass") + result = detect(tmp_path) + all_files = [f for files in result["files"].values() for f in files] + assert not any("dupe.py" in f for f in all_files), "worktree dir was not excluded" + assert any("real.py" in f for f in all_files), "real source was dropped" + + +def test_git_info_exclude_ranks_below_gitignore_negation(tmp_path): + """info/exclude is loaded at lowest priority, so a later .gitignore `!` negation + of the same (non-directory) pattern still wins under last-match-wins (#1810).""" + from graphify.detect import _load_graphifyignore, _is_ignored + (tmp_path / ".git" / "info").mkdir(parents=True) + (tmp_path / ".git" / "info" / "exclude").write_text("secret*.txt\n") + (tmp_path / ".gitignore").write_text("!secret-ok.txt\n") + (tmp_path / "secret-bad.txt").write_text("x") + (tmp_path / "secret-ok.txt").write_text("x") + patterns = _load_graphifyignore(tmp_path) + assert _is_ignored(tmp_path / "secret-bad.txt", tmp_path, patterns) + assert not _is_ignored(tmp_path / "secret-ok.txt", tmp_path, patterns) + + def test_detect_skips_graphify_own_cache(tmp_path): """.graphify/ (extraction cache) must never be re-indexed as source (#873).""" cache = tmp_path / ".graphify" / "cache" @@ -531,6 +825,69 @@ def test_detect_skips_graphify_own_cache(tmp_path): # --- #882: gitignore parent-exclusion rule for ! re-includes --- +def test_anchored_root_wildcard_negation_reincludes_subtree(tmp_path): + """`/*` stays at the root, so `!/src/` makes the subtree walkable (#1975).""" + for rel in ("src/app/main.py", "src/lib/util.py", "docs/guide.md", "README.md"): + path = tmp_path / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("x\n") + (tmp_path / ".graphifyignore").write_text("/*\n!/src/\n") + + result = detect(tmp_path) + + files = { + Path(path).relative_to(tmp_path).as_posix() + for paths in result["files"].values() + for path in paths + } + assert files == {"src/app/main.py", "src/lib/util.py"} + + +def test_anchored_negation_cannot_skip_excluded_parent(tmp_path): + """Re-including a child cannot rescue it while its parent stays excluded.""" + victim = tmp_path / "src" / "app" / "main.py" + victim.parent.mkdir(parents=True) + victim.write_text("x\n") + (tmp_path / ".graphifyignore").write_text("/*\n!/src/app/\n") + + assert detect(tmp_path)["total_files"] == 0 + + +def test_path_pattern_single_star_does_not_cross_segment(tmp_path): + """A regular `*` matches one component; recursive matching requires `**`.""" + direct = tmp_path / "src" / "main.py" + nested = tmp_path / "src" / "app" / "main.py" + nested.parent.mkdir(parents=True) + direct.write_text("x\n") + nested.write_text("x\n") + for pattern in ("/src/*.py", "src/*.py"): + (tmp_path / ".graphifyignore").write_text(f"{pattern}\n") + result = detect(tmp_path) + files = [path for paths in result["files"].values() for path in paths] + assert not any(path.endswith("src/main.py") for path in files) + assert any(path.endswith("src/app/main.py") for path in files) + + +def test_directory_only_negation_does_not_reinclude_file(tmp_path): + """A trailing slash restricts a pattern to directories, as in gitignore.""" + readme = tmp_path / "README.md" + readme.write_text("# docs\n") + (tmp_path / ".graphifyignore").write_text("/*\n!/README.md/\n") + + assert detect(tmp_path)["total_files"] == 0 + + +def test_anchored_double_star_crosses_path_segments(tmp_path): + """`**` retains recursive gitignore matching at zero or more depths.""" + direct = tmp_path / "src" / "generated.py" + nested = tmp_path / "src" / "app" / "deep" / "generated.py" + nested.parent.mkdir(parents=True) + direct.write_text("x\n") + nested.write_text("x\n") + (tmp_path / ".graphifyignore").write_text("/src/**/generated.py\n") + + assert detect(tmp_path)["total_files"] == 0 + def test_negation_cannot_rescue_file_under_excluded_dir(tmp_path): """A ! re-include cannot un-ignore a file whose parent dir is excluded (#882).""" from graphify.detect import _is_ignored, _load_graphifyignore @@ -810,9 +1167,26 @@ def test_sensitive_does_not_flag_tokenizer_py(): def test_sensitive_does_not_flag_tokenize_py(): assert not _is_sensitive(Path("tokenize.py")) -def test_sensitive_flags_passwords_py(): - # passwords.py is just as likely a secret store as passwords.txt — code ext is no excuse - assert _is_sensitive(Path("passwords.py")) +def test_sensitive_does_not_flag_passwords_py(): + # #1666: a programming-language source file named after a domain noun is a + # module, not a secret store. Silently dropping it hid real code from the graph. + # Genuine secret stores are .env/.pem/credentials.json etc. (still flagged below). + assert not _is_sensitive(Path("passwords.py")) + + +def test_sensitive_does_not_flag_ruby_code_modules(): + # #1666 exact cases: Rails source modules with keyword-ish names must survive. + assert not _is_sensitive(Path("app/models/device_token.rb")) + assert not _is_sensitive(Path("app/controllers/api/v1/passwords_controller.rb")) + + +def test_sensitive_still_flags_data_secret_stores(): + # #1666 guard: the exemption is ONLY for real source code, not data/config + # formats — credentials.json / oauth_token.json / secrets.yaml are the secret + # stores Stage 3 must keep catching (even though .json routes through CODE). + assert _is_sensitive(Path("credentials.json")) + assert _is_sensitive(Path("oauth_token.json")) + assert _is_sensitive(Path("app_secret.yaml")) def test_sensitive_flags_ssh_dir(): assert _is_sensitive(Path("/home/user/.ssh/id_rsa")) @@ -848,6 +1222,57 @@ def test_sensitive_token_config_yaml(): assert _is_sensitive(Path("token_config.yaml")) +# ── #1943: Stage 1 dir check gets the same source carve-out as Stage 3 ── +# secrets/ and credentials/ are as often real source packages (Go +# internal/secrets, a credentials/ service module) as credential stores. +# Genuine programming-language source beneath them must be graphed; data and +# config formats — the formats credentials actually ship in — stay dropped, +# and dedicated credential-store dirs (.ssh, .gnupg, .aws, .gcloud) keep +# dropping everything with no carve-out. + +def test_sensitive_does_not_flag_source_under_secrets_dir(): + # #1943 exact cases: real source under ambiguous dir names survives. + assert not _is_sensitive(Path("internal/secrets/vault.go")) + assert not _is_sensitive(Path("app/services/credentials/manager.py")) + +def test_sensitive_still_flags_data_under_secrets_dir(): + # #1943 guard: the carve-out is ONLY for real source — data/config files + # under ambiguous dirs remain flagged, whatever their nesting depth. + assert _is_sensitive(Path("secrets/db.json")) + assert _is_sensitive(Path(".secrets/token.yaml")) + assert _is_sensitive(Path("deploy/credentials/prod.env")) + assert _is_sensitive(Path("internal/secrets/README.md")) # docs are not source + +def test_sensitive_flags_everything_under_credential_store_dirs(): + # #1943: dedicated stores get no carve-out — even source-classified files + # inside .ssh/.gnupg/.aws/.gcloud stay dropped. + assert _is_sensitive(Path("/home/user/.ssh/config")) + assert _is_sensitive(Path(".aws/credentials")) + assert _is_sensitive(Path(".gnupg/helper.py")) + assert _is_sensitive(Path("backup/.gcloud/sync.sh")) + +def test_sensitive_dir_carveout_does_not_bypass_name_screens(): + # #1943: rescued source still falls through to Stages 2-3, so a NON-source + # file whose name/extension is sensitive stays dropped even though the dir + # carve-out spared genuine source beside it. + assert _is_sensitive(Path("credentials/id_rsa")) # extensionless key + assert _is_sensitive(Path("secrets/deploy.pem")) # Stage 2 extension + # #2106: `service_account.py` is real source (e.g. Google's oauth2 lib), not a + # secret. The old unbounded `service.account` substring wrongly dropped it; + # it is now indexed. A downloaded `service-account.json` key still drops. + assert not _is_sensitive(Path("secrets/service_account.py")) + + +def test_sensitive_dir_carveout_still_drops_tfvars_values_store(): + # #1943 follow-up: genuine source under secrets/ is rescued, but .tfvars is + # Terraform's canonical values store (real secrets), not source — it stays + # dropped, while the real code file beside it is kept. + assert _is_sensitive(Path("secrets/prod.tfvars")) + assert not _is_sensitive(Path("secrets/loader.py")) + # .tf / .hcl are genuine infra source and remain graphable under secrets/. + assert not _is_sensitive(Path("secrets/main.tf")) + + # ── Generic keywords must be load-bearing: topic slugs are not secret stores ── # A keyword buried mid-phrase in a >=3-word descriptive name is a note ABOUT # the topic, not a credential file. It must not be silently dropped. @@ -911,6 +1336,42 @@ def test_save_manifest_skips_semantic_hash_for_files_without_cache(tmp_path): assert str(doc2) not in manifest, "failed-chunk file must be absent from manifest" +def test_save_manifest_clear_semantic_erases_stale_hash_for_omitted_file(tmp_path): + """#1948: a file stamped in an earlier run, then omitted from ``files`` on + a later run (LLM dropped its chunk / #1890 retry), must not keep surviving + with its stale semantic_hash from the prior run — the seed loop copies + the on-disk row verbatim otherwise, and detect_incremental(kind='semantic') + reports it unchanged, silently defeating the #1890 retry promise.""" + import json + + doc = tmp_path / "docs" / "doc.md" + doc.parent.mkdir() + doc.write_text("# Doc\n\ncontent") + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + + # Run 1: doc.md is dispatched and stamped. + corpus = {str(doc)} + save_manifest({"document": [str(doc)]}, manifest_path, root=tmp_path, scan_corpus=corpus) + manifest = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + assert manifest["docs/doc.md"]["semantic_hash"] != "" + + # Run 2 (--force re-run): the model omits doc.md this time, so cli.py's + # _stamped_manifest_files() drops it from the files dict passed here — + # but it was still dispatched, so the caller passes it via clear_semantic. + save_manifest( + {"document": []}, manifest_path, root=tmp_path, + scan_corpus=corpus, clear_semantic={str(doc)}, + ) + manifest = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + assert manifest["docs/doc.md"]["semantic_hash"] == "", ( + "omitted file must have its stale semantic_hash cleared, not inherited" + ) + + inc = detect_incremental(tmp_path, manifest_path, kind="semantic") + assert [Path(f).name for f in inc["new_files"]["document"]] == ["doc.md"], ( + "cleared file must be re-queued for semantic extraction" + ) + def test_save_manifest_without_filter_unchanged_for_code(tmp_path): """Code files must be stamped in the manifest regardless of semantic cache.""" @@ -1460,6 +1921,91 @@ def test_detect_incremental_portable_across_paths(tmp_path): ) +def _rewrite_manifest_keys_nfd(manifest_path): + """Rewrite a saved manifest so every key is in NFD form, simulating a + manifest written by a macOS run where os.walk/getcwd yielded decomposed + paths (#2221). Returns the rewritten key list for sanity checks.""" + import json + p = Path(manifest_path) + raw = json.loads(p.read_text(encoding="utf-8")) + nfd = {unicodedata.normalize("NFD", k): v for k, v in raw.items()} + p.write_text(json.dumps(nfd), encoding="utf-8") + return list(nfd) + + +def test_manifest_nfc_keys_survive_macos_path_forms(tmp_path): + """#2221 (portable/relative-key manifest): a manifest whose keys were + written in NFD (macOS os.walk form) must still match an NFC scan, so + --update reports nothing new/changed/deleted instead of re-extracting + the whole corpus. + + NOTE: the fixture filename must contain a character that actually + decomposes under NFD ("é" in "café" does, as do "ä" and "й"). Plain + Cyrillic like "заметка" has no decomposition, so NFC == NFD and the + test would pass vacuously even without the fix. Keep the byte-wise + inequality assertion below when changing the fixture name. + """ + corpus = tmp_path / "corpus" + (corpus / "docs").mkdir(parents=True) + nfc_name = unicodedata.normalize("NFC", "café.md") + assert nfc_name != unicodedata.normalize("NFD", nfc_name) # must decompose + (corpus / "docs" / nfc_name).write_text("hello unicode\n") + + # Manifest lives OUTSIDE the corpus so it never enters the scan. + manifest_path = str(tmp_path / "out" / "manifest.json") + full = detect(corpus) + assert full["total_files"] == 1 # sanity: the café file was scanned + save_manifest(full["files"], manifest_path, root=corpus) + + # Simulate the macOS-written manifest: keys stored in NFD form. + nfd_keys = _rewrite_manifest_keys_nfd(manifest_path) + # Sanity: the on-disk keys are genuinely decomposed, not silently NFC. + assert any(unicodedata.normalize("NFC", k) != k for k in nfd_keys) + + inc = detect_incremental(corpus, manifest_path) + assert inc["new_total"] == 0, ( + f"NFD manifest keys must match NFC scan paths (#2221); " + f"new_files={inc['new_files']}" + ) + assert all(v == [] for v in inc["new_files"].values()) + assert inc["deleted_files"] == [], ( + f"NFD keys misreported as deletions: {inc['deleted_files']}" + ) + assert inc["excluded_files"] == [] + + +def test_manifest_nfc_keys_legacy_absolute(tmp_path): + """#2221 exact repro: legacy manifest saved WITHOUT root (absolute keys), + then rewritten to NFD. Before the load_manifest/detect_incremental NFC + normalization, every file looked simultaneously new AND deleted on + --update. + + NOTE: as above, the filename must contain an NFD-decomposable character + ("é"); a non-decomposing name would make this test vacuous. + """ + corpus = tmp_path / "corpus" + (corpus / "docs").mkdir(parents=True) + nfc_name = unicodedata.normalize("NFC", "café.md") + assert nfc_name != unicodedata.normalize("NFD", nfc_name) # must decompose + (corpus / "docs" / nfc_name).write_text("hello unicode\n") + + manifest_path = str(tmp_path / "out" / "manifest.json") + full = detect(corpus) + assert full["total_files"] == 1 + # No root= -> legacy absolute-keyed manifest format. + save_manifest(full["files"], manifest_path) + + _rewrite_manifest_keys_nfd(manifest_path) + + inc = detect_incremental(corpus, manifest_path) + assert inc["new_total"] == 0, ( + f"legacy absolute NFD keys must match NFC scan (#2221); " + f"new_files={inc['new_files']}" + ) + assert inc["deleted_files"] == [] + assert inc["excluded_files"] == [] + + def test_save_manifest_in_root_symlink_roundtrips(tmp_path): """In-root symlinks must store under the symlink's own name, not the resolved target. Resolving the key when relativizing pointed the stored @@ -1538,3 +2084,449 @@ def test_convert_office_file_does_not_rewrite_existing_sidecar(tmp_path, monkeyp second = detect_mod.convert_office_file(src, out_dir) assert second == first assert second.stat().st_mtime_ns == mtime_before + + +def test_convert_office_file_sidecar_name_stable_across_checkouts(tmp_path, monkeypatch): + """#2059: the sidecar name must depend on the scan-root-RELATIVE path, not the + absolute checkout location, so the same tracked file in two clones/worktrees + produces the same sidecar name (no unbounded duplicates when graphify-out/ is + committed). Also verifies the no-root fallback matches the explicit form.""" + monkeypatch.setattr(detect_mod, "xlsx_to_markdown", lambda p: "sheet body") + + def _sidecar(root): + src = root / "docs" / "report.xlsx" + out_dir = root / "graphify-out" / "converted" + return detect_mod.convert_office_file(src, out_dir, root=root) + + checkout_a = tmp_path / "checkout-a" + checkout_b = tmp_path / "somewhere-else" / "checkout-b" + (checkout_a / "docs").mkdir(parents=True) + (checkout_b / "docs").mkdir(parents=True) + out_a = _sidecar(checkout_a) + out_b = _sidecar(checkout_b) + assert out_a is not None and out_b is not None + assert out_a.name == out_b.name, "sidecar name must be stable across checkouts (#2059)" + assert out_a.parent != out_b.parent # sanity: genuinely different locations + + # No explicit root -> the out_dir.parent.parent fallback yields the same name. + fallback = detect_mod.convert_office_file( + checkout_a / "docs" / "report.xlsx", checkout_a / "graphify-out" / "converted" + ) + assert fallback is not None and fallback.name == out_a.name + + +def test_convert_office_file_hash_disambiguates_same_stem(tmp_path, monkeypatch): + """Two same-stem Office files in different subdirs must still get distinct + sidecar names — the relative-path hash preserves the disambiguation purpose.""" + monkeypatch.setattr(detect_mod, "xlsx_to_markdown", lambda p: "body") + root = tmp_path / "repo" + (root / "a").mkdir(parents=True) + (root / "b").mkdir(parents=True) + out_dir = root / "graphify-out" / "converted" + out_a = detect_mod.convert_office_file(root / "a" / "report.xlsx", out_dir, root=root) + out_b = detect_mod.convert_office_file(root / "b" / "report.xlsx", out_dir, root=root) + assert out_a is not None and out_b is not None + assert out_a.name != out_b.name, "same-stem files in different dirs must differ (#2059)" + + +def test_convert_office_file_outside_root_falls_back(tmp_path, monkeypatch): + """A source outside the scan root (--include, custom layouts) falls back to the + absolute-path hash without raising, and stays deterministic.""" + monkeypatch.setattr(detect_mod, "docx_to_markdown", lambda p: "body") + root = tmp_path / "repo" + (root / "graphify-out" / "converted").mkdir(parents=True) + outside = tmp_path / "elsewhere" / "doc.docx" + out_dir = root / "graphify-out" / "converted" + out1 = detect_mod.convert_office_file(outside, out_dir, root=root) + out2 = detect_mod.convert_office_file(outside, out_dir, root=root) + assert out1 is not None and out1.name == out2.name + + +def test_detect_keeps_env_source_dirs(tmp_path): + """#2058: a real source directory named env/ or *_env/ with no virtualenv + markers must be indexed, not silently pruned as a false-positive venv.""" + src_env = tmp_path / "src_env" + (src_env / "env").mkdir(parents=True) + (src_env / "env" / "ctrl_mem_env.py").write_text("def build_env():\n return 1\n") + (src_env / "other_dir").mkdir() + (src_env / "other_dir" / "also_real.py").write_text("def x():\n return 2\n") + + all_files = [f for files in detect(tmp_path)["files"].values() for f in files] + assert any("ctrl_mem_env.py" in f for f in all_files), "env/ source dir wrongly pruned (#2058)" + assert any("also_real.py" in f for f in all_files), "*_env/ subtree wrongly pruned (#2058)" + + # Nested env/ under a scan root that IS the *_env dir (issue's exact-match case). + nested = [f for files in detect(src_env)["files"].values() for f in files] + assert any("ctrl_mem_env.py" in f for f in nested), "nested env/ pruned when scanned directly (#2058)" + + +def test_detect_still_prunes_real_env_venv(tmp_path): + """#2058: an env/ dir that IS a real virtualenv (has markers) is still pruned, + and the pruned dir is recorded in the traceable pruned_noise_dirs bucket.""" + venv = tmp_path / "env" + (venv / "lib").mkdir(parents=True) + (venv / "pyvenv.cfg").write_text("home = /usr/bin\n") + (venv / "lib" / "sixish.py").write_text("x = 1\n") + (tmp_path / "main.py").write_text("def main():\n return 1\n") + + result = detect(tmp_path) + all_files = [f for files in result["files"].values() for f in files] + assert not any("sixish.py" in f for f in all_files), "real venv env/ must still be pruned" + assert any("main.py" in f for f in all_files) + assert any(f"{os.sep}env{os.sep}" in d for d in result["pruned_noise_dirs"]), ( + "pruned venv must be traceable in pruned_noise_dirs (#2058)" + ) + + +def test_detect_prunes_venv_names_without_markers(tmp_path): + """#2058 must not loosen the unambiguous names: venv/.venv/*_venv are still + pruned by name alone (no markers needed).""" + for name in ("venv", ".venv", "my_venv"): + d = tmp_path / name + d.mkdir() + (d / "mod.py").write_text("y = 1\n") + (tmp_path / "app.py").write_text("def a():\n return 1\n") + all_files = [f for files in detect(tmp_path)["files"].values() for f in files] + assert any("app.py" in f for f in all_files) + for name in ("venv", ".venv", "my_venv"): + assert not any(f"{os.sep}{name}{os.sep}" in f for f in all_files), f"{name} must stay pruned" + + +def test_detect_records_unclassified_extensionless_files(tmp_path): + # #1692: extensionless, non-shebang project files (Dockerfile, Makefile, ...) + # were considered but left no trace. detect() now lists them under + # "unclassified" so they can be surfaced instead of silently vanishing. + (tmp_path / "app.py").write_text("def f():\n return 1\n") + (tmp_path / "Dockerfile").write_text("FROM python:3.12\nRUN pip install x\n") + (tmp_path / "Makefile").write_text("build:\n\techo hi\n") + (tmp_path / "LICENSE").write_text("MIT License\n") + res = detect(tmp_path) + unclassified = sorted(Path(p).name for p in res.get("unclassified", [])) + assert unclassified == ["Dockerfile", "LICENSE", "Makefile"] + # real code is still classified, not swept into unclassified + assert any("app.py" in f for f in res["files"].get("code", [])) + + +def test_detect_unclassified_empty_when_all_supported(tmp_path): + (tmp_path / "a.py").write_text("x = 1\n") + (tmp_path / "README.md").write_text("# hi\n") + res = detect(tmp_path) + assert res.get("unclassified", []) == [] + + +def test_graphifyinclude_is_inert_and_not_unclassified(tmp_path, capsys): + """#2112: .graphifyinclude support was removed (dead since #873). + + A leftover .graphifyinclude must not error, must not surface in the + unclassified list, and must not change which real files are indexed. + detect() prints a one-time stderr note so the removal is not silent. + """ + (tmp_path / "main.py").write_text("x = 1\n") + + baseline = detect(tmp_path) + capsys.readouterr() # discard any baseline output + + (tmp_path / ".graphifyinclude").write_text(".github/\ndocs/**\n") + result = detect(tmp_path) + + # not surfaced as an unclassified scan input + assert not any(".graphifyinclude" in p for p in result["unclassified"]) + # real files are indexed exactly as before; the file changes nothing + assert result["files"] == baseline["files"] + assert any("main.py" in f for f in result["files"]["code"]) + # one-time stderr note, matching the [graphify] warning convention + err = capsys.readouterr().err + assert err.count("[graphify] WARNING: .graphifyinclude is no longer supported") == 1 + + +def test_detect_reports_walk_errors_key(): + """detect() always surfaces a walk_errors list so callers can tell whether + enumeration was complete.""" + import tempfile + d = Path(tempfile.mkdtemp()) + (d / "a.py").write_text("def f(): pass\n") + res = detect(d) + assert "walk_errors" in res + assert res["walk_errors"] == [] + + +def test_detect_surfaces_unreadable_dir_instead_of_silent_skip(tmp_path, capsys): + """os.walk silently skips a subtree whose scandir raises (permissions, or a + dir deleted mid-walk); that under-enumeration used to be invisible and could + yield a silently partial graph. detect() now records it in walk_errors and + warns, while still enumerating the rest of the tree.""" + import os + if os.geteuid() == 0: + import pytest + pytest.skip("running as root: chmod 000 does not block scandir") + (tmp_path / "a.py").write_text("def f(): pass\n") + locked = tmp_path / "locked" + locked.mkdir() + (locked / "b.py").write_text("def g(): pass\n") + os.chmod(locked, 0o000) + try: + res = detect(tmp_path) + finally: + os.chmod(locked, 0o755) # restore for cleanup + code = res["files"]["code"] + assert any(f.endswith("a.py") for f in code) # rest of tree still enumerated + assert len(res["walk_errors"]) >= 1 + assert "could not scan" in capsys.readouterr().err + + +def test_nested_gitignore_star_does_not_ignore_outside_its_dir(tmp_path): + """A nested .gitignore containing a bare `*` (auto-written by e.g. the + hypothesis library into .hypothesis/) must ignore ONLY that directory's + contents — matching it against root-relative paths ignored the entire + corpus (detect() returned 0 files on a real repo). Regression for #1873.""" + (tmp_path / "README.md").write_text("# hello") + (tmp_path / "main.py").write_text("x = 1") + hyp = tmp_path / ".hypothesis" + hyp.mkdir() + (hyp / ".gitignore").write_text("*\n") + (hyp / "cached.py").write_text("y = 2") + + result = detect(tmp_path) + + assert result["total_files"] == 2 # README.md + main.py survive; .hypothesis/* ignored + + +def test_nested_gitignore_patterns_still_apply_inside_their_dir(tmp_path): + """Counterpart guard: the anchor-scoped fix must not stop nested ignore + files from working WITHIN their own subtree.""" + (tmp_path / "main.py").write_text("x = 1") + sub = tmp_path / "sub" + sub.mkdir() + (sub / ".gitignore").write_text("*.log\n") + (sub / "keep.py").write_text("y = 2") + (sub / "noise.log").write_text("z") + + result = detect(tmp_path) + + assert result["total_files"] == 2 # main.py + sub/keep.py; sub/noise.log ignored + + +def test_nested_gitignore_does_not_govern_sibling_project(tmp_path): + """A nested .gitignore ('data/') in one project must not drop a sibling + project's data/ files, and the drop must be recorded in the `ignored` + diagnostic field rather than silently vanishing (#1922).""" + (tmp_path / "run.py").write_text("x = 1") + pa = tmp_path / "project_a" / "data" + pa.mkdir(parents=True) + (pa / "loader.py").write_text("def load(): pass") + pb = tmp_path / "project_b" + (pb / "data").mkdir(parents=True) + (pb / ".gitignore").write_text("data/\n") + (pb / "data" / "dump.csv").write_text("a,b\n1,2\n") + + result = detect(tmp_path) + + all_paths = [f for v in result["files"].values() for f in v] + assert any( + f.endswith(os.path.join("project_a", "data", "loader.py")) for f in all_paths + ), "sibling project_a/data/loader.py must survive project_b's nested ignore" + assert not any(f.endswith("dump.csv") for f in all_paths) + # The legitimately-ignored subtree is recorded, not silently dropped. + assert any( + e.rstrip(os.sep).endswith(os.path.join("project_b", "data")) + for e in result["ignored"] + ), f"ignored subtree should be recorded in detect()['ignored']: {result['ignored']}" + + +# --------------------------------------------------------------------------- +# #1908: manifest must not retain scan-excluded files as permanent +# "deleted" entries. Full-scan saves prune excluded-but-alive rows; subset +# saves keep preserving untouched rows (#917); out-of-root rows never prune. +# --------------------------------------------------------------------------- + +def test_save_manifest_full_scan_prunes_excluded_but_alive_row(tmp_path): + """A row for a file that still exists on disk but left the scan corpus + (newly excluded) is dropped when the caller passes the full corpus.""" + import json + a = tmp_path / "a.py" + b = tmp_path / "b.py" + a.write_text("x = 1\n") + b.write_text("y = 2\n") + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + + save_manifest({"code": [str(a), str(b)]}, manifest_path, root=tmp_path) + raw = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + assert set(raw) == {"a.py", "b.py"} + + # Second full scan no longer covers b.py (excluded), yet b.py is alive. + save_manifest( + {"code": [str(a)]}, manifest_path, root=tmp_path, + scan_corpus={str(a)}, + ) + raw = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + assert set(raw) == {"a.py"}, ( + f"excluded-but-alive row must be pruned on a full-scan save, got {set(raw)}" + ) + + +def test_save_manifest_full_scan_still_prunes_missing_file(tmp_path): + """Genuine deletions keep being pruned when scan_corpus is passed.""" + import json + a = tmp_path / "a.py" + gone = tmp_path / "gone.py" + a.write_text("x = 1\n") + gone.write_text("y = 2\n") + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + save_manifest({"code": [str(a), str(gone)]}, manifest_path, root=tmp_path) + + gone.unlink() + save_manifest( + {"code": [str(a)]}, manifest_path, root=tmp_path, + scan_corpus={str(a)}, + ) + raw = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + assert set(raw) == {"a.py"} + + +def test_save_manifest_subset_save_preserves_untouched_rows(tmp_path): + """Without scan_corpus (changed_paths hooks, skill runbooks, #917) a + subset save must keep seeding rows for files it wasn't given.""" + import json + a = tmp_path / "a.py" + b = tmp_path / "b.py" + a.write_text("x = 1\n") + b.write_text("y = 2\n") + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + save_manifest({"code": [str(a), str(b)]}, manifest_path, root=tmp_path) + + # Incremental hook re-stamps only a.py; b.py's row must survive. + save_manifest({"code": [str(a)]}, manifest_path, root=tmp_path) + raw = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + assert set(raw) == {"a.py", "b.py"}, ( + f"subset saves must preserve untouched rows (#917), got {set(raw)}" + ) + + +def test_save_manifest_full_scan_keeps_out_of_root_rows(tmp_path): + """Out-of-root entries (--include sources, symlinked corpora) are never + walked by detect, so their absence from the corpus is not exclusion + evidence — a full-scan save must keep them.""" + import json + a = tmp_path / "a.py" + a.write_text("x = 1\n") + outside = tmp_path.parent / f"{tmp_path.name}-extern.py" + outside.write_text("z = 3\n") + try: + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + save_manifest( + {"code": [str(a), str(outside)]}, manifest_path, root=tmp_path + ) + save_manifest( + {"code": [str(a)]}, manifest_path, root=tmp_path, + scan_corpus={str(a)}, + ) + raw = json.loads(Path(manifest_path).read_text(encoding="utf-8")) + assert "a.py" in raw + assert str(outside.resolve()) in raw, ( + f"out-of-root rows must never be pruned to the scan, got {set(raw)}" + ) + finally: + outside.unlink(missing_ok=True) + + +def test_detect_incremental_reports_excluded_not_deleted(tmp_path): + """A previously-indexed file that becomes excluded (still on disk) must + land in excluded_files, not deleted_files (#1908).""" + a = tmp_path / "a.py" + b = tmp_path / "b.py" + a.write_text("x = 1\n") + b.write_text("y = 2\n") + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + full = detect(tmp_path) + save_manifest(full["files"], manifest_path, root=tmp_path) + + inc = detect_incremental( + tmp_path, manifest_path, extra_excludes=["b.py"] + ) + assert inc["deleted_files"] == [], ( + f"excluded-but-alive file misreported as deleted: {inc['deleted_files']}" + ) + assert [Path(f).name for f in inc["excluded_files"]] == ["b.py"] + + +def test_detect_incremental_still_reports_real_deletions(tmp_path): + """Counterpart: a manifest row whose file is gone from disk stays in + deleted_files.""" + a = tmp_path / "a.py" + b = tmp_path / "b.py" + a.write_text("x = 1\n") + b.write_text("y = 2\n") + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + full = detect(tmp_path) + save_manifest(full["files"], manifest_path, root=tmp_path) + + b.unlink() + inc = detect_incremental(tmp_path, manifest_path) + assert [Path(f).name for f in inc["deleted_files"]] == ["b.py"] + assert inc["excluded_files"] == [] + + +def test_detect_incremental_exclusion_stable_across_runs(tmp_path): + """After a full-scan save prunes the excluded row, later incremental runs + report the file neither as deleted nor as excluded — the exclusion has + fully settled instead of resurfacing forever.""" + a = tmp_path / "a.py" + b = tmp_path / "b.py" + a.write_text("x = 1\n") + b.write_text("y = 2\n") + manifest_path = str(tmp_path / "graphify-out" / "manifest.json") + full = detect(tmp_path) + save_manifest(full["files"], manifest_path, root=tmp_path) + + # Run 1: b.py newly excluded — reported as excluded, then the full-scan + # save (what extract does at the end of the run) prunes its row. + inc1 = detect_incremental(tmp_path, manifest_path, extra_excludes=["b.py"]) + assert [Path(f).name for f in inc1["excluded_files"]] == ["b.py"] + assert inc1["deleted_files"] == [] + corpus = {f for flist in inc1["files"].values() for f in flist} + save_manifest(inc1["files"], manifest_path, root=tmp_path, scan_corpus=corpus) + + # Run 2 (and beyond): steady state — nothing deleted, nothing excluded. + inc2 = detect_incremental(tmp_path, manifest_path, extra_excludes=["b.py"]) + assert inc2["deleted_files"] == [] + assert inc2["excluded_files"] == [] + + +# ── #2106: sensitive-filter over-match (prose/source rescued, real secrets kept) ── + +@pytest.mark.parametrize("path", [ + "wiki/privacy-tokens.md", # reporter's own hub node + "wiki/ai-token-economics.md", + "wiki/chain-of-hope-tokenomics.md", + "tokenizer.py", + "secretary.py", + "google/oauth2/service_account.py", # real Google auth source + "docs/service-account-setup.md", + "wiki/aws_credentials_rotation_guide.md", + "token.economics.notes.md", # multi-dot topic slug + "password-reset/design.md", +]) +def test_sensitive_filter_indexes_topic_prose_and_source(path): + from graphify.detect import _is_sensitive + assert not _is_sensitive(Path(path)), f"{path} is a topic doc / real source, must be indexed (#2106)" + + +@pytest.mark.parametrize("path", [ + ".env", "id_rsa", "credentials.json", "server.pem", "certs/server.key", + "secrets.md", "passwords.md", "token.md", "token.txt", "api_token.json", + "service-account.json", # a downloaded GCP key file + ".npmrc", ".pypirc", "secring.gpg", ".git-credentials", # #2106 newly-caught + "Secrets/creds.json", "SECRETS/db.json", "ID_RSA", # #2106 case variants + "secrets/prod.tfvars", "credentials/id_rsa", +]) +def test_sensitive_filter_still_excludes_real_secrets(path): + from graphify.detect import _is_sensitive + assert _is_sensitive(Path(path)), f"{path} is a real secret, must stay excluded (#2106)" + + +def test_sensitive_bare_keyword_prose_still_dropped(): + """A prose file whose stem IS exactly a bare keyword still reads as a dump.""" + from graphify.detect import _is_sensitive + assert _is_sensitive(Path("secrets.md")) + assert _is_sensitive(Path("token.rst")) + assert not _is_sensitive(Path("token-lifecycle.md")) # multi-word slug indexed diff --git a/tests/test_dotnet.py b/tests/test_dotnet.py index eca02b09e..ec48f8b75 100644 --- a/tests/test_dotnet.py +++ b/tests/test_dotnet.py @@ -45,6 +45,35 @@ def test_sln_project_dependency(): assert "imports" in _relations(r) +def test_sln_solution_folder_ids_are_relative(tmp_path): + """Solution folders are virtual groupings, not files. Their node ids must be + derived from the folder name only — never the resolved absolute scan path, + which would leak the local username into a committed graph.json (#1789).""" + sln = tmp_path / "App.sln" + sln.write_text( + 'Microsoft Visual Studio Solution File, Format Version 12.00\n' + # a solution folder: type GUID 2150E333-... , name == path, no real file + 'Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Plugins", "Plugins", ' + '"{11111111-1111-1111-1111-111111111111}"\n' + 'EndProject\n' + # a real project resolves to an absolute path as before + 'Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "App", "App\\App.csproj", ' + '"{22222222-2222-2222-2222-222222222222}"\n' + 'EndProject\n', + encoding="utf-8", + ) + r = extract_sln(sln) + assert "error" not in r + # The virtual solution folder must be keyed off its name, with no trace of the + # absolute scan path. (Real-file nodes — the .sln and .csproj — legitimately + # carry absolute ids here; the CLI's id-relativization pass remaps those, but + # never the virtual folder, which is why the leak had to be fixed at source.) + folder = next(n for n in r["nodes"] if n["label"] == "Plugins") + assert folder["id"] == "plugins" + assert folder["source_file"] == "Plugins" + assert str(tmp_path) not in folder["id"] + + # ── .slnx ──────────────────────────────────────────────────────────────────── def test_slnx_extracts_projects(): @@ -97,6 +126,35 @@ def test_csproj_project_references(): assert len(imports) == 6 # 4 packages + 2 project refs +def test_csproj_out_of_root_reference_id_is_portable(tmp_path): + """#1899: a ProjectReference to a project OUTSIDE the scan root must not leak + the absolute scan path (including the OS username) into the node id or + source_file. The out-of-root target gets a portable, `ext_`-namespaced id and + a walk-up relative source_file rather than the absolute-derived form.""" + web = tmp_path / "WebApi"; web.mkdir() + core = tmp_path / "Core"; core.mkdir() + (core / "Core.csproj").write_text( + '' + 'net8.0' + ) + (web / "WebApi.csproj").write_text( + '' + '' + ) + result = extract([web / "WebApi.csproj"], cache_root=web) + marker = str(tmp_path) + for n in result["nodes"]: + assert marker not in n["id"], f"absolute path leaked into id: {n}" + assert marker not in (n.get("source_file") or ""), f"leaked into source_file: {n}" + for e in result["edges"]: + for f in ("source", "target", "source_file"): + assert marker not in str(e.get(f, "")), f"leaked into edge {f}: {e}" + core_ref = [n for n in result["nodes"] if "core" in n["id"].lower()] + assert core_ref, "out-of-root Core reference node missing" + assert core_ref[0]["id"].startswith("ext_") + assert core_ref[0]["source_file"] == "../Core/Core.csproj" + + def test_csproj_target_framework(): r = extract_csproj(FIXTURES / "sample.csproj") assert "net8.0" in _labels(r) @@ -215,6 +273,40 @@ def test_xaml_prism_autowire_false_does_not_infer_from_filename(tmp_path): assert _view_model_edges(r) == [] +def test_xaml_cs_scan_prunes_noise_dirs_and_stays_bounded(tmp_path): + """The code-behind/.cs scan prunes noise dirs (node_modules/.venv/.git/...) + during traversal and is bounded, so it links the real ViewModel while a decoy + .cs buried in node_modules is never scanned — and it can't rglob a huge tree + and hang (the standalone-root escape that stalled the suite).""" + proj = tmp_path / "App" + (proj / "Views").mkdir(parents=True) + (proj / "ViewModels").mkdir() + (proj / "App.csproj").write_text('', encoding="utf-8") + (proj / "Views" / "MainWindow.xaml").write_text( + '\n' + ' \n' + "\n", encoding="utf-8") + (proj / "ViewModels" / "MainWindowViewModel.cs").write_text( + "namespace App.ViewModels { public class MainWindowViewModel {} }\n", encoding="utf-8") + # A decoy with the SAME class name inside a noise dir: if pruning failed it + # would be scanned and make the link ambiguous/wrong. + nm = proj / "node_modules" / "pkg" + nm.mkdir(parents=True) + (nm / "Decoy.cs").write_text( + "namespace App.ViewModels { public class MainWindowViewModel {} }\n", encoding="utf-8") + r = extract_xaml(proj / "Views" / "MainWindow.xaml") + assert "error" not in r + nodes = {n["id"]: n for n in r["nodes"]} + edges = _view_model_edges(r) + assert len(edges) == 1 + tgt = nodes[edges[0]["target"]] + assert tgt["label"] == "MainWindowViewModel" + assert "node_modules" not in (tgt.get("source_file") or ""), "decoy in node_modules was scanned" + + def test_xaml_links_communitytoolkit_generated_members_and_event_to_command(): r = extract_xaml(FIXTURES / "xaml_viewmodel" / "Views" / "ToolkitView.xaml") nodes = {n["id"]: n for n in r["nodes"]} diff --git a/tests/test_evidence_binding.py b/tests/test_evidence_binding.py new file mode 100644 index 000000000..6df068184 --- /dev/null +++ b/tests/test_evidence_binding.py @@ -0,0 +1,216 @@ +"""Tests for semantic evidence-binding in graphify.llm. + +A code node the model returns whose symbol name has no evidence in the dispatched +source is flagged ``verification = "unverified"`` (never dropped). This closes +the intra-file hallucination gap that ``_out_of_scope`` (#1895) — which only +rejects nodes attributed to a file that was NOT dispatched — cannot see. +""" + +from pathlib import Path +from unittest.mock import patch + +from graphify import llm + + +_SOURCE = ( + "def real_function():\n" + " return PaymentProcessor().charge_card()\n" + "\n" + "class PaymentProcessor:\n" + " def charge_card(self):\n" + " pass\n" +) + + +def _run(files, nodes, tmp_path): + """Drive extract_files_direct with a faked backend returning ``nodes``.""" + result = { + "nodes": nodes, + "edges": [], + "hyperedges": [], + "input_tokens": 1, + "output_tokens": 1, + "finish_reason": "stop", + } + with patch("graphify.llm._call_openai_compat", return_value=result): + return llm.extract_files_direct(files, backend="kimi", api_key="k", root=tmp_path) + + +def _by_label(out): + return {n["label"]: n for n in out["nodes"]} + + +def test_fabricated_code_symbol_is_downgraded(tmp_path): + src = tmp_path / "mod.py" + src.write_text(_SOURCE, encoding="utf-8") + nodes = [ + {"id": "a", "label": "real_function()", "file_type": "code", "source_file": "mod.py"}, + {"id": "b", "label": "totally_fabricated_symbol()", "file_type": "code", "source_file": "mod.py"}, + {"id": "c", "label": "Payments Overview", "file_type": "concept", "source_file": "mod.py"}, + ] + out = _by_label(_run([src], nodes, tmp_path)) + # The fabricated symbol has no evidence in the source -> flagged. + assert out["totally_fabricated_symbol()"]["verification"] == "unverified" + # A symbol that IS in the source is verified -> untouched (no confidence key). + assert "verification" not in out["real_function()"] + # A concept node is prose, never checked. + assert "verification" not in out["Payments Overview"] + + +def test_qualified_and_prettified_labels_do_not_false_positive(tmp_path): + src = tmp_path / "mod.py" + src.write_text(_SOURCE, encoding="utf-8") + nodes = [ + {"id": "a", "label": "PaymentProcessor.charge_card()", "file_type": "code", "source_file": "mod.py"}, + {"id": "b", "label": "charge_card(amount, token)", "file_type": "code", "source_file": "mod.py"}, + ] + out = _by_label(_run([src], nodes, tmp_path)) + # Any label identifier present in the source verifies the whole label. + assert "verification" not in out["PaymentProcessor.charge_card()"] + assert "verification" not in out["charge_card(amount, token)"] + + +def test_document_and_sourceless_nodes_are_never_flagged(tmp_path): + src = tmp_path / "mod.py" + src.write_text(_SOURCE, encoding="utf-8") + nodes = [ + {"id": "a", "label": "Nonexistent Heading", "file_type": "document", "source_file": "mod.py"}, + {"id": "b", "label": "orphan_symbol()", "file_type": "code"}, # no source_file + ] + out = _by_label(_run([src], nodes, tmp_path)) + assert "verification" not in out["Nonexistent Heading"] + assert "verification" not in out["orphan_symbol()"] + + +def test_node_attributed_to_undispatched_file_is_left_to_out_of_scope(tmp_path): + src = tmp_path / "mod.py" + src.write_text(_SOURCE, encoding="utf-8") + # other.py exists on disk but is NOT dispatched in this call. + (tmp_path / "other.py").write_text("def elsewhere():\n pass\n", encoding="utf-8") + nodes = [ + {"id": "a", "label": "ghost_func()", "file_type": "code", "source_file": "other.py"}, + ] + out = _by_label(_run([src], nodes, tmp_path)) + # Not in the dispatched set -> #1895's domain, not evidence-binding's. + assert "verification" not in out["ghost_func()"] + + +def test_uncheckable_short_label_is_not_flagged(tmp_path): + src = tmp_path / "mod.py" + src.write_text(_SOURCE, encoding="utf-8") + nodes = [ + {"id": "a", "label": "id()", "file_type": "code", "source_file": "mod.py"}, + ] + out = _by_label(_run([src], nodes, tmp_path)) + # "id" is < 3 chars, so there is no checkable identifier -> leave as-is. + assert "verification" not in out["id()"] + + +def test_existing_lower_confidence_is_not_overwritten(tmp_path): + src = tmp_path / "mod.py" + src.write_text(_SOURCE, encoding="utf-8") + nodes = [ + {"id": "a", "label": "made_up()", "file_type": "code", "source_file": "mod.py", "confidence": "INFERRED"}, + ] + out = _by_label(_run([src], nodes, tmp_path)) + # The model already hedged it (INFERRED); evidence-binding leaves it entirely + # alone — no verification flag added, and its confidence is never clobbered. + assert out["made_up()"]["confidence"] == "INFERRED" + assert "verification" not in out["made_up()"] + + +def test_label_identifiers_helper(): + assert llm._label_identifiers("foo()") == ["foo"] + assert llm._label_identifiers("Cls.method(x)") == ["Cls", "method"] + assert llm._label_identifiers("id()") == [] # all tokens < 3 chars + assert llm._label_identifiers("") == [] + + +def test_bind_node_evidence_returns_downgrade_count(tmp_path): + src = tmp_path / "mod.py" + src.write_text(_SOURCE, encoding="utf-8") + result = { + "nodes": [ + {"id": "a", "label": "real_function()", "file_type": "code", "source_file": "mod.py"}, + {"id": "b", "label": "fake_one()", "file_type": "code", "source_file": "mod.py"}, + {"id": "c", "label": "fake_two()", "file_type": "code", "source_file": "mod.py"}, + ], + } + downgraded = llm._bind_node_evidence(result, [src], tmp_path) + assert downgraded == 2 + + +def test_evidence_binding_handles_file_slice(tmp_path): + # A slice reports its PARENT file as source_file; verification runs against + # the slice bytes the model actually saw. + from graphify.file_slice import FileSlice + + src = tmp_path / "big.md" + src.write_text("intro\n" + _SOURCE + "\ntail\n", encoding="utf-8") + text = src.read_text() + fs = FileSlice(path=src, start=0, end=len(text), index=0, total=1) + result = { + "nodes": [ + {"id": "a", "label": "real_function()", "file_type": "code", "source_file": "big.md"}, + {"id": "b", "label": "ghost_symbol()", "file_type": "code", "source_file": "big.md"}, + ], + } + n = llm._bind_node_evidence(result, [fs], tmp_path) + by = {x["label"]: x for x in result["nodes"]} + assert n == 1 + assert "verification" not in by["real_function()"] + assert by["ghost_symbol()"]["verification"] == "unverified" + + +def test_evidence_binding_handles_absolute_source_file(tmp_path): + src = tmp_path / "mod.py" + src.write_text(_SOURCE, encoding="utf-8") + # Model returns an absolute source_file rather than a root-relative one. + result = {"nodes": [ + {"id": "a", "label": "ghost_symbol()", "file_type": "code", "source_file": str(src)}, + ]} + n = llm._bind_node_evidence(result, [src], tmp_path) + assert n == 1 + assert result["nodes"][0]["verification"] == "unverified" + + +def test_downgrade_emits_stderr_summary(tmp_path, capsys): + src = tmp_path / "mod.py" + src.write_text(_SOURCE, encoding="utf-8") + nodes = [{"id": "b", "label": "totally_made_up_symbol()", "file_type": "code", "source_file": "mod.py"}] + _run([src], nodes, tmp_path) + err = capsys.readouterr().err + assert "unverified" in err + + +def test_unverified_flag_does_not_fail_validation(): + # The flag lives on its own ``verification`` field, deliberately NOT on the + # validated ``confidence`` key (whose vocabulary is edge-only), so it must + # never make an otherwise-valid node fail validation. + from graphify.validate import validate_extraction + + extraction = { + "nodes": [{"id": "n1", "label": "foo", "file_type": "code", + "source_file": "a.md", "verification": "unverified"}], + "edges": [], + } + errors = validate_extraction(extraction) + assert not any("verification" in str(e).lower() for e in errors) + + +def test_diagnostics_reports_unverified_node_count(): + # The consumer: diagnose_extraction surfaces the persisted verification flag + # so it is not a dead field. + from graphify.diagnostics import diagnose_extraction, format_diagnostic_report + + extraction = { + "nodes": [ + {"id": "n1", "label": "ghost", "file_type": "code", + "source_file": "a.md", "verification": "unverified"}, + {"id": "n2", "label": "real", "file_type": "code", "source_file": "a.md"}, + ], + "edges": [], + } + summary = diagnose_extraction(extraction) + assert summary["unverified_node_count"] == 1 + assert "unverified_code_nodes: 1" in format_diagnostic_report(summary) diff --git a/tests/test_explain_cli.py b/tests/test_explain_cli.py index 7759f30e3..94ba22b9c 100644 --- a/tests/test_explain_cli.py +++ b/tests/test_explain_cli.py @@ -123,3 +123,116 @@ def test_explain_no_lesson_line_for_unannotated_node(monkeypatch, tmp_path, caps p = _write_graph(tmp_path) out = _run(monkeypatch, p, "validateSanitySession", capsys) assert "Lesson:" not in out + + +def test_explain_connection_shows_call_site_line(monkeypatch, tmp_path, capsys): + """BUG1: an explain connection shows the edge's call-SITE line (in the + caller's file), not the caller's def line.""" + graph_data = { + "directed": False, "multigraph": False, "graph": {}, + "nodes": [ + {"id": "loader", "label": "load_state()", + "source_file": "apollo.py", "source_location": "L90", "community": 0}, + {"id": "trans", "label": "transition_state()", + "source_file": "state.py", "source_location": "L56", "community": 0}, + ], + "links": [ + {"source": "loader", "target": "trans", "relation": "calls", + "confidence": "EXTRACTED", "source_file": "apollo.py", "source_location": "L158"}, + ], + } + p = tmp_path / "graph.json" + p.write_text(json.dumps(graph_data)) + out = _run(monkeypatch, p, "transition_state", capsys) + # The inbound caller line must cite the call site apollo.py:L158. + caller_line = next(l for l in out.splitlines() if "<-- load_state()" in l) + assert "apollo.py:L158" in caller_line, f"call site missing from: {caller_line!r}" + assert "apollo.py:L90" not in caller_line # never the caller's def line + # The queried node's own header still shows its def line (correct). + assert "state.py" in out and "L56" in out + + +# --- #2009: high-degree nodes must not silently hide the cut connections ------ + +def _write_high_degree_graph(tmp_path, n_callers=30, files=None): + """A node with n_callers callers, spread across `files` (default: 3 + files, so counts land above 1 per file and truncation kicks in — the + CLI shows the top 20 by neighbor degree, cutting the rest).""" + files = files or ["app/handlers/email.py", "app/jobs/retry.py", "lib/workers/queue.py"] + nodes = [{"id": "hub", "label": "hub()", + "source_file": "lib/hub.py", "community": 0}] + links = [] + for i in range(n_callers): + fpath = files[i % len(files)] + nid = f"caller_{i}" + nodes.append({"id": nid, "label": f"caller_{i}()", + "source_file": fpath, "community": 0}) + links.append({"source": nid, "target": "hub", "relation": "calls", + "confidence": "EXTRACTED", "source_file": fpath, + "source_location": f"L{10 + i}"}) + graph_data = {"directed": False, "multigraph": False, "graph": {}, + "nodes": nodes, "links": links} + p = tmp_path / "graph.json" + p.write_text(json.dumps(graph_data)) + return p + + +def test_explain_truncation_notice_present_for_high_degree_node(monkeypatch, tmp_path, capsys): + """Baseline: the cut count is still announced (pre-existing behavior).""" + p = _write_high_degree_graph(tmp_path, n_callers=30) + out = _run(monkeypatch, p, "hub", capsys) + assert "Connections (30):" in out + assert "... and 10 more" in out + + +def test_explain_groups_cut_callers_by_file_instead_of_dropping_them(monkeypatch, tmp_path, capsys): + """#2009: past the top-20 cutoff, the remaining callers must still be + accounted for — grouped by file with counts — instead of vanishing + behind a bare '... and N more'. No caller may be lost silently: the + per-file counts in the aggregation must sum back to the cut total.""" + p = _write_high_degree_graph( + tmp_path, n_callers=30, + files=["app/handlers/email.py", "app/jobs/retry.py", "lib/workers/queue.py"], + ) + out = _run(monkeypatch, p, "hub", capsys) + assert "Grouped by file:" in out + assert "<-- lib/workers/queue.py: 4 connections" in out + assert "<-- app/handlers/email.py: 3 connections" in out + assert "<-- app/jobs/retry.py: 3 connections" in out + # No silent loss: the aggregated counts must sum to the announced cut. + grouped_lines = [ + l for l in out.splitlines() if l.strip().startswith(("<--", "-->")) and "connection" in l + ] + total = sum(int(l.rsplit(":", 1)[1].split()[0]) for l in grouped_lines) + assert total == 10 # 30 connections - 20 shown = 10 cut, all accounted for + + +def test_explain_no_grouping_section_when_under_cutoff(monkeypatch, tmp_path, capsys): + """Regression guard: nodes at or below the 20-connection cutoff keep the + pre-#2009 output byte-for-byte (no new section, no behavior change).""" + p = _write_high_degree_graph(tmp_path, n_callers=5) + out = _run(monkeypatch, p, "hub", capsys) + assert "Grouped by file:" not in out + assert "more" not in out + + +def test_explain_grouping_boundary_at_exactly_21_vs_20_connections(monkeypatch, tmp_path, capsys): + """Pin the exact `> 20` cutoff itself. The other #2009 tests use 30 and 5 + connections, both comfortably clear of the edge — nothing here fails if a + future refactor shifts the boundary by one. One node at exactly 21 + connections (one past the cutoff) must show the grouped section with + exactly one grouped entry; one node at exactly 20 (at the cutoff, not + past it) must show neither.""" + p21 = _write_high_degree_graph(tmp_path, n_callers=21, files=["lib/only.py"]) + out21 = _run(monkeypatch, p21, "hub", capsys) + assert "Grouped by file:" in out21 + assert "<-- lib/only.py: 1 connection" in out21 + grouped_lines21 = [ + l for l in out21.splitlines() if l.strip().startswith(("<--", "-->")) and "connection" in l + ] + assert len(grouped_lines21) == 1 # exactly one grouped entry, not zero, not more + + p20 = _write_high_degree_graph(tmp_path, n_callers=20, files=["lib/only.py"]) + out20 = _run(monkeypatch, p20, "hub", capsys) + assert "Grouped by file:" not in out20 + assert "more" not in out20 diff --git a/tests/test_export.py b/tests/test_export.py index be4743bc5..28a4707a7 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -99,6 +99,48 @@ def test_to_graphml_tolerates_none_attribute_values(): content = out.read_text() assert " 0 + + +def test_existing_graph_node_count(tmp_path): + from graphify.export import existing_graph_node_count, MALFORMED_GRAPH + p = tmp_path / "graph.json" + assert existing_graph_node_count(p) is None # absent -> nothing to protect + p.write_text("", encoding="utf-8") + assert existing_graph_node_count(p) is None # empty -> nothing to protect + # Non-empty but unparseable must fail CLOSED (sentinel), matching to_json's + # #479 guard — a corrupt/mid-write file could be hiding a complete graph. + p.write_text("{not json", encoding="utf-8") + assert existing_graph_node_count(p) is MALFORMED_GRAPH # malformed -> fail closed + p.write_text('{"nodes": "notalist"}', encoding="utf-8") + assert existing_graph_node_count(p) is MALFORMED_GRAPH # structurally wrong -> fail closed + p.write_text('{"nodes": [{"id": "a"}, {"id": "b"}], "links": []}', encoding="utf-8") + assert existing_graph_node_count(p) == 2 # valid diff --git a/tests/test_extract.py b/tests/test_extract.py index 4e76ad120..e0a880b89 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -1,7 +1,12 @@ import json import os +import sys from collections import Counter from pathlib import Path + +import pytest + +from graphify.build import build_from_json from graphify.extract import extract_python, extract, collect_files, _make_id, extract_bash, extract_json, _DISPATCH FIXTURES = Path(__file__).parent / "fixtures" @@ -100,6 +105,37 @@ def test_extract_disambiguates_duplicate_symbol_ids_by_source_path(tmp_path): assert edge["target"] in node_ids, f"Dangling structural target: {edge}" +def test_cpp_unresolved_base_class_stubs_stay_disambiguated_by_file(tmp_path): + """Two different files' same-named, otherwise-undefined base class must not + collapse onto one shared stub node. + + The C++ base_class_clause handler used to build its stub inline instead of + calling ensure_named_node(), so it never tagged the stub with origin_file. + Without that tag, _disambiguate_colliding_node_ids couldn't tell file A's + reference to unresolved `Base` apart from file B's, and every file's + unresolved base class merged onto one bare id -- which could then collide + with an unrelated same-named real definition anywhere else in the corpus. + """ + first = tmp_path / "a" / "Foo.cpp" + second = tmp_path / "b" / "Bar.cpp" + first.parent.mkdir(parents=True) + second.parent.mkdir(parents=True) + first.write_text("class Foo : public Base {};\n", encoding="utf-8") + second.write_text("class Bar : public Base {};\n", encoding="utf-8") + + result = extract([first, second], cache_root=tmp_path) + base_stubs = [ + node for node in result["nodes"] + if node["label"] == "Base" and not node.get("source_file") + ] + assert len(base_stubs) == 2 + assert len({node["id"] for node in base_stubs}) == 2 + + inherits_edges = [e for e in result["edges"] if e["relation"] == "inherits"] + assert len(inherits_edges) == 2 + assert len({e["target"] for e in inherits_edges}) == 2 + + def test_cross_file_type_annotation_refs_resolve_to_single_node(tmp_path): """#1402: a class defined once but referenced via type annotations in N other files must NOT create 1+N phantom duplicate nodes (with the referencing file's @@ -934,6 +970,337 @@ def test_python_qualified_class_method_call_resolves_extracted(tmp_path): assert call_edges[0]["confidence"] == "EXTRACTED" +def test_degenerate_symbol_name_does_not_leak_absolute_id(tmp_path): + """#1899 variant B: a symbol whose name normalizes to nothing (a minified `$` + function, a JSONC `"//"` key) must not be minted — `_make_id(stem, "")` + collapses to the bare, absolute-path-derived file stem, leaking the scan path + and colliding with the file node. Such nodes carry no graph signal.""" + (tmp_path / "vendor.js").write_text( + "function $(){return 1}\nfunction real(){return 2}\n", encoding="utf-8" + ) + result = extract([tmp_path / "vendor.js"], cache_root=tmp_path) + marker = str(tmp_path) + for n in result["nodes"]: + assert marker not in n["id"], f"absolute path leaked into id: {n}" + labels = {n.get("label") for n in result["nodes"]} + assert "real()" in labels, "the real function must still be extracted" + assert "$()" not in labels, "the degenerate `$` symbol must be dropped (#1899)" + + +def test_out_of_tree_cache_root_keeps_source_file_relative_to_scan_root(tmp_path): + """#1941: `--out ` must not basename every in-root node. + + The CLI passes cache_root= to relocate the cache, but that value also + anchored relativization, so every scanned file failed `relative_to(root)`, fell + into `_portable_out_of_root_sf`, tripped the `updepth > 3` walk-up guard meant + for stray out-of-root ProjectReferences, and collapsed to a bare basename. + An explicit `root=` anchors ids/source_file on the SCAN root regardless of + where the cache lives. + """ + scan_root = tmp_path / "corpus" + nested = scan_root / "src" / "Data" / "Database" / "RepositoryTests" + nested.mkdir(parents=True) + (nested / "order_repository_tests.py").write_text( + "class OrderRepositoryTests:\n def test_get(self):\n return 1\n", + encoding="utf-8", + ) + # >3 levels off the shared ancestor: the exact shape that triggered basenaming. + out_dir = tmp_path / "a" / "b" / "c" / "d" / "out" + out_dir.mkdir(parents=True) + + result = extract( + [nested / "order_repository_tests.py"], + cache_root=out_dir, + root=scan_root, + ) + source_files = { + n["source_file"] for n in result["nodes"] if n.get("source_file") + } + assert source_files, "expected nodes carrying a source_file" + assert source_files == { + "src/Data/Database/RepositoryTests/order_repository_tests.py" + }, f"source_file must stay relative to the scan root, got {source_files}" + # The point of the field: it resolves back to a real file against the root. + for sf in source_files: + assert (scan_root / sf).is_file(), f"{sf} does not resolve under {scan_root}" + # #1899 must not regress: no absolute path / username leak. + for n in result["nodes"]: + assert str(tmp_path) not in (n.get("source_file") or "") + assert str(tmp_path) not in n["id"] + + +def test_python_module_qualified_call_resolves_extracted(tmp_path): + """`module.func()` where `module` is imported resolves to the callable that + module contains, with an EXTRACTED `calls` edge (#1883). A lowercase module + receiver was previously dropped alongside instance calls.""" + mathlib = tmp_path / "mathlib.py" + caller = tmp_path / "caller.py" + mathlib.write_text("def compute(x):\n return x * 2\n") + caller.write_text( + "import mathlib\n\n" + "def use_qualified(n):\n" + " return mathlib.compute(n)\n" + ) + result = extract([caller, mathlib], cache_root=tmp_path) + nodes = {n["id"]: n for n in result["nodes"]} + edges = [ + e for e in result["edges"] + if e["relation"] == "calls" + and "use_qualified" in nodes[e["source"]]["label"] + and "compute" in nodes[e["target"]]["label"] + and "mathlib.py" in (nodes[e["target"]].get("source_file") or "") + ] + assert len(edges) == 1, f"expected one use_qualified->compute edge, got {edges}" + assert edges[0]["confidence"] == "EXTRACTED" + + +def test_python_module_qualified_call_requires_the_import(tmp_path): + """A `module.func()` call must resolve only against a module the caller's own + file imports — a local instance `o.compute()` (o is a parameter) must NOT be + linked to a same-named function in some other module (#1883 false-edge guard).""" + mathlib = tmp_path / "mathlib.py" + caller = tmp_path / "caller.py" + mathlib.write_text("def compute(x):\n return x * 2\n") + # no `import mathlib`; `o` is just a parameter that happens to expose compute() + caller.write_text("def via_obj(o):\n return o.compute(3)\n") + result = extract([caller, mathlib], cache_root=tmp_path) + nodes = {n["id"]: n for n in result["nodes"]} + bad = [ + e for e in result["edges"] + if e["relation"] == "calls" + and "via_obj" in nodes[e["source"]]["label"] + and "compute" in nodes[e["target"]]["label"] + ] + assert bad == [], f"non-imported receiver must not link cross-file: {bad}" + + +def test_python_from_import_alias_module_call_resolves(tmp_path): + """`from pkg import mod as alias` must resolve `alias.func()` the same way the + unaliased `from pkg import mod` / `mod.func()` form already does (#2082). The + local alias binding was untracked, so the aliased receiver never matched the + submodule's own stem and the `calls` edge silently disappeared while the + file-level `imports_from` edge stayed present and made the graph look intact. + + Also covers the fix's `local_alias` hint hygiene: like the existing + `target_file` transient hint (#1814), it must be popped once the resolver + that reads it has run, never surviving into the returned edges/graph.json -- + otherwise an internal local-variable name from the source tree leaks into + every graph.json produced from an aliased import.""" + pkg = tmp_path / "pkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("") + (pkg / "gate.py").write_text("def validate(rows):\n return bool(rows)\n") + caller = pkg / "caller.py" + caller.write_text( + "from pkg import gate as m_gate\n\n" + "def use_alias(rows):\n" + " return m_gate.validate(rows)\n" + ) + result = extract( + [caller, pkg / "gate.py", pkg / "__init__.py"], + cache_root=tmp_path, + root=tmp_path, + ) + nodes = {n["id"]: n for n in result["nodes"]} + edges = [ + e for e in result["edges"] + if e["relation"] == "calls" + and "use_alias" in nodes[e["source"]]["label"] + and "validate" in nodes[e["target"]]["label"] + and "gate.py" in (nodes[e["target"]].get("source_file") or "") + ] + assert len(edges) == 1, f"expected one use_alias->validate edge, got {edges}" + assert edges[0]["confidence"] == "EXTRACTED" + leaked = [e for e in result["edges"] if "local_alias" in e] + assert leaked == [], f"local_alias hint must not survive into the output: {leaked}" + + +def test_python_import_as_alias_module_call_resolves(tmp_path): + """`import mod as alias` must resolve `alias.func()` the same way `import mod` + / `mod.func()` already does (#1883) -- the same untracked-alias regression as + `from pkg import mod as alias` (#2082), on the plain `import` form.""" + mathlib = tmp_path / "mathlib.py" + caller = tmp_path / "caller.py" + mathlib.write_text("def compute(x):\n return x * 2\n") + caller.write_text( + "import mathlib as m\n\n" + "def use_aliased_import(n):\n" + " return m.compute(n)\n" + ) + result = extract([caller, mathlib], cache_root=tmp_path) + nodes = {n["id"]: n for n in result["nodes"]} + edges = [ + e for e in result["edges"] + if e["relation"] == "calls" + and "use_aliased_import" in nodes[e["source"]]["label"] + and "compute" in nodes[e["target"]]["label"] + and "mathlib.py" in (nodes[e["target"]].get("source_file") or "") + ] + assert len(edges) == 1, f"expected one use_aliased_import->compute edge, got {edges}" + assert edges[0]["confidence"] == "EXTRACTED" + + +def test_python_try_except_from_import_alias_module_call_resolves(tmp_path): + """The issue's own motivating shape (#2082): `from pkg import mod as alias` + guarded by a `try:`/`except ImportError:` fallback assignment, the pattern + real code uses for an optional dependency. The issue explicitly called out + that the drop is independent of the `try:` nesting -- this locks that in as + a regression test rather than relying only on the unwrapped module-level + form above.""" + pkg = tmp_path / "pkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("") + (pkg / "gate.py").write_text("def validate(rows):\n return bool(rows)\n") + caller = pkg / "caller_try.py" + caller.write_text( + "try:\n" + " from pkg import gate as t_gate\n" + "except ImportError:\n" + " t_gate = None\n\n" + "def use_try_alias(rows):\n" + " return t_gate.validate(rows)\n" + ) + result = extract( + [caller, pkg / "gate.py", pkg / "__init__.py"], + cache_root=tmp_path, + root=tmp_path, + ) + nodes = {n["id"]: n for n in result["nodes"]} + edges = [ + e for e in result["edges"] + if e["relation"] == "calls" + and "use_try_alias" in nodes[e["source"]]["label"] + and "validate" in nodes[e["target"]]["label"] + and "gate.py" in (nodes[e["target"]].get("source_file") or "") + ] + assert len(edges) == 1, f"expected one use_try_alias->validate edge, got {edges}" + assert edges[0]["confidence"] == "EXTRACTED" + + +def test_python_dotted_import_alias_module_call_resolves(tmp_path): + """`import pkg.mod as alias` -- the dotted absolute-import form the issue + flagged as needing coverage -- must resolve `alias.func()` the same way the + single-segment `import mathlib as m` form above does. This exercises the + `aliased_import` branch of `_import_python`'s `import_statement` arm with a + multi-segment module name, where the target id comes from collapsing + `pkg.gate` rather than a bare stem.""" + pkg = tmp_path / "pkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("") + (pkg / "gate.py").write_text("def validate(rows):\n return bool(rows)\n") + caller = pkg / "caller_dotted.py" + caller.write_text( + "import pkg.gate as g_alias\n\n" + "def use_dotted_alias(rows):\n" + " return g_alias.validate(rows)\n" + ) + result = extract( + [caller, pkg / "gate.py", pkg / "__init__.py"], + cache_root=tmp_path, + root=tmp_path, + ) + nodes = {n["id"]: n for n in result["nodes"]} + edges = [ + e for e in result["edges"] + if e["relation"] == "calls" + and "use_dotted_alias" in nodes[e["source"]]["label"] + and "validate" in nodes[e["target"]]["label"] + and "gate.py" in (nodes[e["target"]].get("source_file") or "") + ] + assert len(edges) == 1, f"expected one use_dotted_alias->validate edge, got {edges}" + assert edges[0]["confidence"] == "EXTRACTED" + + +def test_python_relative_from_import_alias_module_call_resolves(tmp_path): + """`from . import mod as alias` -- a relative sibling-module import with an + alias -- must resolve `alias.func()` the same way the absolute `from pkg + import mod as alias` form above does. Relative imports route through the + same #1146 submodule-import path (module_imports' local_name slot) with a + level instead of an absolute module name, which is a distinct branch from + the absolute-import case already covered.""" + pkg = tmp_path / "pkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("") + (pkg / "gate.py").write_text("def validate(rows):\n return bool(rows)\n") + caller = pkg / "caller_relative.py" + caller.write_text( + "from . import gate as r_gate\n\n" + "def use_relative_alias(rows):\n" + " return r_gate.validate(rows)\n" + ) + result = extract( + [caller, pkg / "gate.py", pkg / "__init__.py"], + cache_root=tmp_path, + root=tmp_path, + ) + nodes = {n["id"]: n for n in result["nodes"]} + edges = [ + e for e in result["edges"] + if e["relation"] == "calls" + and "use_relative_alias" in nodes[e["source"]]["label"] + and "validate" in nodes[e["target"]]["label"] + and "gate.py" in (nodes[e["target"]].get("source_file") or "") + ] + assert len(edges) == 1, f"expected one use_relative_alias->validate edge, got {edges}" + assert edges[0]["confidence"] == "EXTRACTED" + + +def test_python_external_aliased_import_fabricates_no_call_edge(tmp_path): + """#2082 must not over-resolve: an aliased import of an EXTERNAL/uncorpus + module (`import numpy as np; np.array()`) has no in-corpus callee, so it must + produce NO `calls` edge — the alias resolution stays inside the member-call + carve-out (in-corpus target required).""" + caller = tmp_path / "app.py" + caller.write_text( + "import numpy as np\n" + "from os import path as p\n\n" + "def build(rows):\n" + " p.join('a', 'b')\n" + " return np.array(rows)\n" + ) + result = extract([caller], cache_root=tmp_path, root=tmp_path) + nodes = {n["id"]: n for n in result["nodes"]} + fabricated = [ + e for e in result["edges"] + if e["relation"] in ("calls", "indirect_call") + and ("array" in nodes.get(e["target"], {}).get("label", "") + or "join" in nodes.get(e["target"], {}).get("label", "")) + ] + assert fabricated == [], f"external aliased calls must not fabricate edges: {fabricated}" + + +def test_python_aliased_call_survives_warm_cache(tmp_path): + """#2082: the aliased `calls` edge must survive a warm (cache-hit) re-extract. + The fix threads a transient `local_alias` hint that is popped after the + resolver runs; the per-file cache must serialize it BEFORE the pop, or the + edge would resolve only on a cold run and silently vanish on the next.""" + pkg = tmp_path / "pkg" + pkg.mkdir() + (pkg / "__init__.py").write_text("") + (pkg / "gate.py").write_text("def validate(rows):\n return bool(rows)\n") + caller = pkg / "caller.py" + caller.write_text( + "from pkg import gate as m_gate\n\n" + "def use_alias(rows):\n" + " return m_gate.validate(rows)\n" + ) + paths = [caller, pkg / "gate.py", pkg / "__init__.py"] + + def _alias_edges(result): + nodes = {n["id"]: n for n in result["nodes"]} + return [ + e for e in result["edges"] + if e["relation"] == "calls" + and "use_alias" in nodes[e["source"]]["label"] + and "validate" in nodes[e["target"]]["label"] + ] + + cold = extract(paths, cache_root=tmp_path, root=tmp_path) + assert len(_alias_edges(cold)) == 1, "cold run must resolve the aliased call" + warm = extract(paths, cache_root=tmp_path, root=tmp_path) # cache-hit + assert len(_alias_edges(warm)) == 1, "aliased call edge vanished on warm cache (#2082)" + + def test_python_qualified_call_resolves_when_method_name_collides_with_caller(tmp_path): """The real #1446 shape: a viewset action `approve()` delegates to a SERVICE action of the SAME name via `Service.approve()`. The bare-name in-file lookup @@ -1078,7 +1445,7 @@ def test_extract_falls_back_to_sequential_when_parallel_returns_false(tmp_path, calls = {"parallel": 0, "sequential": 0} real_sequential = extract_mod._extract_sequential - def fake_parallel(uncached_work, per_file, effective_root, max_workers, total_files): + def fake_parallel(uncached_work, per_file, root, max_workers, total_files, cache_location=None): calls["parallel"] += 1 return False # simulate the post-fix BrokenProcessPool branch @@ -1121,6 +1488,63 @@ def submit(self, *a, **kw): assert "__main__" in out, "warning must hint at the Windows __main__ guard idiom" +def test_extract_parallel_skips_pool_when_max_workers_is_one(tmp_path, monkeypatch): + """#2173: a resolved worker count of 1 must not spawn a ProcessPoolExecutor. + + The Windows post-commit hook exports GRAPHIFY_MAX_WORKERS=1, so before this the + rebuild spawned a one-worker pool for >= _PARALLEL_THRESHOLD files: no + parallelism, one process spawn plus an IPC round trip per file, and the only + window where the parent's rebuild watchdog (os._exit) can orphan a worker + mid-task. _extract_parallel must decline (return False) so the caller extracts + sequentially in-process. + """ + import concurrent.futures + from graphify import extract as extract_mod + + spawned = {"count": 0} + + def fake_pool(*args, **kwargs): + spawned["count"] += 1 + raise AssertionError("ProcessPoolExecutor must not be constructed for 1 worker") + + monkeypatch.setattr(concurrent.futures, "ProcessPoolExecutor", fake_pool) + monkeypatch.setenv("GRAPHIFY_MAX_WORKERS", "1") + + uncached = [(i, FIXTURES / "sample.py") for i in range(25)] # >= _PARALLEL_THRESHOLD + per_file: list = [None] * len(uncached) + + ok = extract_mod._extract_parallel(uncached, per_file, tmp_path, None, len(uncached)) + assert ok is False, "must hand the work back for sequential extraction" + assert spawned["count"] == 0, "no pool may be spawned when max_workers resolves to 1" + + +def test_extract_parallel_still_spawns_pool_for_multiple_workers(tmp_path, monkeypatch): + """Guard the #2173 skip: >1 worker must still take the pool path.""" + import concurrent.futures + from graphify import extract as extract_mod + + spawned = {"count": 0} + + class FakePool: + def __init__(self, *a, **kw): + spawned["count"] += 1 + def __enter__(self): + return self + def __exit__(self, *a): + return False + def submit(self, *a, **kw): + raise concurrent.futures.process.BrokenProcessPool("stop here") + + monkeypatch.setattr(concurrent.futures, "ProcessPoolExecutor", FakePool) + monkeypatch.setenv("GRAPHIFY_MAX_WORKERS", "4") + + uncached = [(i, FIXTURES / "sample.py") for i in range(25)] + per_file: list = [None] * len(uncached) + + extract_mod._extract_parallel(uncached, per_file, tmp_path, None, len(uncached)) + assert spawned["count"] == 1, "multi-worker runs must still use the pool" + + # --------------------------------------------------------------------------- # Bash extractor tests (#866) # --------------------------------------------------------------------------- @@ -1174,6 +1598,126 @@ def test_extract_bash_emits_source_imports_from(tmp_path): assert import_edges[0].get("context") == "import" +def test_extract_bash_source_via_variable_path_resolves_to_real_file(tmp_path): + """`source "${DIR}/lib/x.sh"` (the `dirname "${BASH_SOURCE[0]}"` idiom) must + resolve to the real file node relative to the script dir — never emit a dead + id baking in the literal `${DIR}` text (#2079).""" + lib = tmp_path / "lib" + lib.mkdir() + helper = lib / "gpu-discover.sh" + helper.write_text("# helper\n", encoding="utf-8") + script = tmp_path / "bench.sh" + script.write_text( + '#!/bin/bash\n' + 'BENCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"\n' + 'source "${BENCH_DIR}/lib/gpu-discover.sh"\n', + encoding="utf-8", + ) + result = extract_bash(script) + import_edges = [e for e in result["edges"] if e["relation"] == "imports_from"] + targets = [e["target"] for e in import_edges] + assert _make_id(str(helper.resolve())) in targets, import_edges + assert not any("$" in t for t in targets), f"dead expansion id emitted: {targets}" + inferred = next(e for e in import_edges + if e["target"] == _make_id(str(helper.resolve()))) + assert inferred.get("confidence") == "INFERRED" + assert inferred.get("context") == "import" + + +def test_extract_bash_source_via_variable_path_no_match_emits_no_dead_edge(tmp_path): + """A variable-built source path with no matching file on disk must emit no + import edge at all — not an `imports` edge to an id containing `${VAR}` (#2079).""" + script = tmp_path / "bench.sh" + script.write_text( + '#!/bin/bash\nsource "${BENCH_DIR}/lib/missing.sh"\n', + encoding="utf-8", + ) + result = extract_bash(script) + edges = [e for e in result["edges"] + if e["relation"] in ("imports", "imports_from")] + assert edges == [], f"variable source with no on-disk match must emit no edge; got: {edges}" + + +@pytest.mark.parametrize("command", ["./helpers.sh", "bash ./helpers.sh"]) +def test_extract_bash_emits_script_invocation_calls(tmp_path, command): + helpers = tmp_path / "helpers.sh" + helpers.write_text("#!/bin/bash\necho helper\n", encoding="utf-8") + script = tmp_path / "deploy.sh" + script.write_text(f"#!/bin/bash\n{command}\n", encoding="utf-8") + + result = extract_bash(script) + invocation = [ + edge for edge in result["edges"] + if edge.get("relation") == "calls" and edge.get("context") == "script_invocation" + ] + + assert invocation == [{ + "source": _make_id(str(script)) + "__entry", + "target": _make_id(str(helpers.resolve())) + "__entry", + "relation": "calls", + "confidence": "EXTRACTED", + "source_file": str(script), + "source_location": "L2", + "weight": 1.0, + "context": "script_invocation", + }] + + +def test_extract_bash_skips_missing_and_shadowed_script_invocations(tmp_path): + helpers = tmp_path / "helpers.sh" + helpers.write_text("#!/bin/bash\necho helper\n", encoding="utf-8") + script = tmp_path / "deploy.sh" + script.write_text( + "#!/bin/bash\n" + "bash() { echo custom; }\n" + "bash ./helpers.sh\n" + "./missing.sh\n", + encoding="utf-8", + ) + + result = extract_bash(script) + + assert not any(edge.get("context") == "script_invocation" for edge in result["edges"]) + + +def test_extract_bash_skips_dynamic_script_invocation(tmp_path): + helpers = tmp_path / "helpers.sh" + helpers.write_text("#!/bin/bash\necho helper\n", encoding="utf-8") + script = tmp_path / "deploy.sh" + script.write_text('#!/bin/bash\nbash "./$SCRIPT.sh"\n', encoding="utf-8") + + result = extract_bash(script) + + assert not any(edge.get("context") == "script_invocation" for edge in result["edges"]) + + +def test_extract_bash_relative_script_invocation_targets_existing_entrypoint(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + helpers = Path("helpers.sh") + helpers.write_text("#!/bin/bash\necho helper\n", encoding="utf-8") + script = Path("deploy.sh") + script.write_text("#!/bin/bash\n./helpers.sh\n", encoding="utf-8") + + result = extract([script, helpers], cache_root=tmp_path, parallel=False) + node_ids = {node["id"] for node in result["nodes"]} + invocation = next(edge for edge in result["edges"] if edge.get("context") == "script_invocation") + + assert invocation["target"] in node_ids + + +def test_extract_bash_attributes_script_invocation_to_function(tmp_path): + helpers = tmp_path / "helpers.sh" + helpers.write_text("#!/bin/bash\necho helper\n", encoding="utf-8") + script = tmp_path / "deploy.sh" + script.write_text("#!/bin/bash\ndeploy() { bash ./helpers.sh; }\n", encoding="utf-8") + + result = extract_bash(script) + deploy = next(node for node in result["nodes"] if node["label"] == "deploy()") + invocation = next(edge for edge in result["edges"] if edge.get("context") == "script_invocation") + + assert invocation["source"] == deploy["id"] + + def test_extract_bash_no_self_loops(): result = extract_bash(FIXTURES / "sample.sh") for e in result["edges"]: @@ -1395,6 +1939,308 @@ def test_extract_bash_source_user_defined_emits_calls_not_imports_from(tmp_path) ) +def test_extract_bash_emits_raw_calls_and_bash_sources_for_sourced_calls(tmp_path): + """extract_bash must surface the data cross-file resolution needs: a + ``bash_sources`` entry per sourced file and a ``raw_calls`` entry for each + call whose callee is not defined in the same file. Without these, + resolve_bash_source_edges has nothing to resolve a sourced-function call + from (#2141).""" + (tmp_path / "b.sh").write_text("#!/usr/bin/env bash\nb_func() { echo ok; }\n") + a = tmp_path / "a.sh" + a.write_text( + "#!/usr/bin/env bash\n" + "source ./b.sh\n" + "main() { b_func; }\n" + ) + result = extract_bash(a) + + sources = result.get("bash_sources", []) + assert any(str(s.get("target_path", "")).endswith("b.sh") for s in sources), sources + + main_nid = next(n["id"] for n in result["nodes"] if n.get("label") == "main()") + raw_calls = result.get("raw_calls", []) + assert any( + rc.get("language") == "bash" + and rc.get("callee") == "b_func" + and rc.get("caller_nid") == main_nid + for rc in raw_calls + ), raw_calls + + +def test_extract_bash_call_to_sourced_function_resolves(tmp_path): + """#2141 repro: a call to a function defined in a sourced file must produce a + real ``calls`` edge through the full extract() pipeline, so ``path`` and + ``callers`` can traverse it.""" + (tmp_path / "b.sh").write_text("#!/usr/bin/env bash\nb_func() { echo ok; }\n") + (tmp_path / "a.sh").write_text( + "#!/usr/bin/env bash\n" + "source ./b.sh\n" + "main() { b_func; }\n" + ) + result = extract([tmp_path / "a.sh", tmp_path / "b.sh"], cache_root=tmp_path) + calls = {(e["source"], e["target"]) for e in result["edges"] if e["relation"] == "calls"} + assert ("a_main", "b_b_func") in calls, sorted(calls) + + +def test_extract_bash_sourced_call_does_not_duplicate_source_edge(tmp_path): + """Wiring the source-backed call resolver must not re-emit the ``imports_from`` + source edge the extractor already resolved for ``source ./b.sh`` (#2141).""" + (tmp_path / "b.sh").write_text("#!/usr/bin/env bash\nb_func() { echo ok; }\n") + (tmp_path / "a.sh").write_text( + "#!/usr/bin/env bash\n" + "source ./b.sh\n" + "main() { b_func; }\n" + ) + result = extract([tmp_path / "a.sh", tmp_path / "b.sh"], cache_root=tmp_path) + imports = [(e["source"], e["target"]) for e in result["edges"] + if e["relation"] == "imports_from"] + assert imports.count(("a", "b")) == 1, imports + + +def test_extract_bash_call_to_external_command_stays_unlinked(tmp_path): + """A call to a command that is not a function in any sourced file (an external + binary) must not gain a cross-file ``calls`` edge — even when a same-named + function exists in an *unsourced* file. Source-scoped resolution is what keeps + #2141 from over-connecting the graph (acceptance criterion).""" + # b.sh is NOT sourced by a.sh, yet defines a function named `deploy`. + (tmp_path / "b.sh").write_text("#!/usr/bin/env bash\ndeploy() { echo ok; }\n") + (tmp_path / "a.sh").write_text( + "#!/usr/bin/env bash\n" + "main() { deploy; }\n" + ) + result = extract([tmp_path / "a.sh", tmp_path / "b.sh"], cache_root=tmp_path) + calls = {(e["source"], e["target"]) for e in result["edges"] if e["relation"] == "calls"} + assert ("a_main", "b_deploy") not in calls, sorted(calls) + + +def test_extract_bash_call_into_extensionless_sourced_lib_resolves(tmp_path): + """#2171: a sourced lib with a bash shebang but no extension must resolve. + + _SHEBANG_DISPATCH already routes an extensionless `#!/usr/bin/env bash` file to + extract_bash, so its functions are indexed, but the cross-file source pass + selected participants by filename suffix only — so the lib was left out and + calls into it never bound. + """ + lib = tmp_path / "mylib" + lib.write_text("#!/usr/bin/env bash\nlib_helper() { echo ok; }\n", encoding="utf-8") + (tmp_path / "a.sh").write_text( + "#!/usr/bin/env bash\n" + "source ./mylib\n" + "main() { lib_helper; }\n", + encoding="utf-8", + ) + result = extract([tmp_path / "a.sh", lib], cache_root=tmp_path) + calls = {(e["source"], e["target"]) for e in result["edges"] if e["relation"] == "calls"} + assert ("a_main", "mylib_lib_helper") in calls, sorted(calls) + + +def test_extract_bash_bare_source_name_resolves_to_sibling(tmp_path): + """#2171: `source lib.sh` with no ./ prefix must bind to the sibling file. + + Only the ``./``/``/``-prefixed branch recorded bash_sources; a bare name fell + through to the opaque ``imports`` fallback, so neither the source edge nor + calls into the lib resolved even though the file sits next to the script. + """ + (tmp_path / "lib.sh").write_text( + "#!/usr/bin/env bash\nbare_helper() { echo ok; }\n", encoding="utf-8" + ) + (tmp_path / "a.sh").write_text( + "#!/usr/bin/env bash\n" + "source lib.sh\n" + "main() { bare_helper; }\n", + encoding="utf-8", + ) + result = extract([tmp_path / "a.sh", tmp_path / "lib.sh"], cache_root=tmp_path) + imports = [(e["source"], e["target"]) for e in result["edges"] + if e["relation"] == "imports_from"] + assert ("a", "lib") in imports, imports + calls = {(e["source"], e["target"]) for e in result["edges"] if e["relation"] == "calls"} + assert ("a_main", "lib_bare_helper") in calls, sorted(calls) + + +def test_extract_bash_bare_source_missing_file_fabricates_nothing(tmp_path): + """The #2171 bare-name branch keeps the existence gate: a name that resolves to + no sibling must not produce an imports_from edge or a bash_sources entry.""" + script = tmp_path / "a.sh" + script.write_text("#!/usr/bin/env bash\nsource nope.sh\n", encoding="utf-8") + result = extract_bash(script) + assert result["bash_sources"] == [], result["bash_sources"] + imports_from = [e for e in result["edges"] if e["relation"] == "imports_from"] + assert imports_from == [], imports_from + + +def test_bash_var_sourced_function_call_resolves(tmp_path): + """End-to-end integration of #2079 + #2141 (#2157/#2139): a library sourced + via the canonical ``${VAR}`` idiom must feed ``bash_sources`` so that + resolve_bash_source_edges binds calls into its functions — not just the + imports_from source edge. Before the extractor appended the resolved path to + ``bash_sources`` in the ``${VAR}`` branch, main() -> util_fn() produced no + calls edge at all.""" + # realpath: tempfile on macOS hands out /var/... which symlinks to + # /private/var/...; the extractor stores the *resolved* target path, so the + # scan root must be the resolved form too or ids anchor inconsistently. + root = Path(os.path.realpath(tmp_path)) + lib = root / "lib" + lib.mkdir() + (lib / "util.sh").write_text( + "#!/usr/bin/env bash\nutil_fn() { :; }\n", encoding="utf-8" + ) + (root / "bench.sh").write_text( + '#!/usr/bin/env bash\n' + 'BENCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"\n' + 'source "${BENCH_DIR}/lib/util.sh"\n' + "main() { util_fn; }\n", + encoding="utf-8", + ) + result = extract( + [root / "bench.sh", lib / "util.sh"], cache_root=root, root=root + ) + + main_id = next( + n["id"] for n in result["nodes"] + if n["label"] == "main()" and n["source_file"].endswith("bench.sh") + ) + util_id = next( + n["id"] for n in result["nodes"] + if n["label"] == "util_fn()" and n["source_file"].endswith("util.sh") + ) + sourced_calls = [ + e for e in result["edges"] + if e["relation"] == "calls" + and e["source"] == main_id and e["target"] == util_id + ] + assert len(sourced_calls) == 1, ( + f"expected exactly one calls edge {main_id} -> {util_id}; got: " + f"{sorted((e['source'], e['target']) for e in result['edges'] if e['relation'] == 'calls')}" + ) + # The source edge itself must still be there alongside the call binding. + imports = {(e["source"], e["target"]) for e in result["edges"] + if e["relation"] == "imports_from"} + assert ("bench", "lib_util") in imports, sorted(imports) + + +def test_extract_bash_source_suffix_guard_mid_path_variable(tmp_path): + """`source "lib/${X}.sh"` keeps an expansion in the suffix, so the + ``$``-in-suffix guard of _bash_source_suffix must reject it: no + imports/imports_from edge and no bash_sources entry may be fabricated.""" + lib = tmp_path / "lib" + lib.mkdir() + (lib / "extras.sh").write_text("#!/usr/bin/env bash\n", encoding="utf-8") + script = tmp_path / "run.sh" + script.write_text( + '#!/usr/bin/env bash\nsource "lib/${X}.sh"\n', encoding="utf-8" + ) + result = extract_bash(script) + fabricated = [e for e in result["edges"] + if e["relation"] in ("imports", "imports_from")] + assert fabricated == [], fabricated + assert result["bash_sources"] == [], result["bash_sources"] + + +def test_extract_bash_source_suffix_guard_whole_variable_path(tmp_path): + """`source "$CONFIG_FILE"` strips to an empty suffix — nothing literal is + left to resolve, so no edge and no bash_sources entry may be emitted.""" + (tmp_path / "config.sh").write_text("#!/usr/bin/env bash\n", encoding="utf-8") + script = tmp_path / "run.sh" + script.write_text( + '#!/usr/bin/env bash\nsource "$CONFIG_FILE"\n', encoding="utf-8" + ) + result = extract_bash(script) + fabricated = [e for e in result["edges"] + if e["relation"] in ("imports", "imports_from")] + assert fabricated == [], fabricated + assert result["bash_sources"] == [], result["bash_sources"] + + +def test_extract_bash_source_suffix_guard_rejects_traversal(tmp_path): + """`source "${D}/../secret.sh"` must hit the ``..`` guard. The target file + exists one level up, so without the guard the suffix WOULD resolve and + fabricate both the edge and the bash_sources entry.""" + (tmp_path / "secret.sh").write_text( + "#!/usr/bin/env bash\nleak() { :; }\n", encoding="utf-8" + ) + scripts = tmp_path / "scripts" + scripts.mkdir() + script = scripts / "run.sh" + script.write_text( + '#!/usr/bin/env bash\nsource "${D}/../secret.sh"\n', encoding="utf-8" + ) + result = extract_bash(script) + fabricated = [e for e in result["edges"] + if e["relation"] in ("imports", "imports_from")] + assert fabricated == [], fabricated + assert result["bash_sources"] == [], result["bash_sources"] + + +def test_extract_bash_var_source_uses_tracked_assignment_base(tmp_path): + """#2172: `${VAR}` must resolve against the variable's tracked base. + + #2079 always resolved the literal suffix against the script's own directory. + That is right for `DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"`, but + when the variable points elsewhere -- here ROOT is the script dir's parent -- + and a same-named decoy exists under the script dir, the edge bound to the + decoy: a wrong edge to a real node. + """ + (tmp_path / "lib").mkdir() + (tmp_path / "lib" / "utils.sh").write_text( + "#!/usr/bin/env bash\nreal_util() { echo real; }\n", encoding="utf-8" + ) + scripts = tmp_path / "scripts" + (scripts / "lib").mkdir(parents=True) + decoy = scripts / "lib" / "utils.sh" + decoy.write_text( + "#!/usr/bin/env bash\ndecoy_util() { echo decoy; }\n", encoding="utf-8" + ) + script = scripts / "deploy.sh" + script.write_text( + '#!/usr/bin/env bash\n' + 'ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"\n' + 'source "${ROOT}/lib/utils.sh"\n', + encoding="utf-8", + ) + result = extract_bash(script) + targets = [str(s["target_path"]) for s in result["bash_sources"]] + assert targets, "the ${VAR} source must still resolve" + for t in targets: + assert Path(t).resolve() == (tmp_path / "lib" / "utils.sh").resolve(), t + assert Path(t).resolve() != decoy.resolve(), f"bound to the decoy: {t}" + + +def test_extract_bash_var_source_script_dir_idiom_still_resolves(tmp_path): + """The canonical script-dir idiom must keep working (#2079 regression guard).""" + (tmp_path / "lib").mkdir() + (tmp_path / "lib" / "x.sh").write_text( + "#!/usr/bin/env bash\nx_fn() { :; }\n", encoding="utf-8" + ) + script = tmp_path / "bench.sh" + script.write_text( + '#!/usr/bin/env bash\n' + 'BENCH_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"\n' + 'source "${BENCH_DIR}/lib/x.sh"\n', + encoding="utf-8", + ) + result = extract_bash(script) + targets = [Path(s["target_path"]).resolve() for s in result["bash_sources"]] + assert (tmp_path / "lib" / "x.sh").resolve() in targets, targets + + +def test_extract_bash_var_source_untracked_var_keeps_script_dir_guess(tmp_path): + """An untracked variable (assigned from the environment, or not assigned in + this file at all) keeps the #2079 script-dir guess rather than binding + nowhere -- the fallback must survive the #2172 change.""" + lib = tmp_path / "lib" + lib.mkdir() + (lib / "y.sh").write_text("#!/usr/bin/env bash\ny_fn() { :; }\n", encoding="utf-8") + script = tmp_path / "run.sh" + script.write_text( + '#!/usr/bin/env bash\nsource "${SOME_EXTERNAL_DIR}/lib/y.sh"\n', + encoding="utf-8", + ) + result = extract_bash(script) + targets = [Path(s["target_path"]).resolve() for s in result["bash_sources"]] + assert (lib / "y.sh").resolve() in targets, targets + + # --------------------------------------------------------------------------- # JSON extractor tests (#866) # --------------------------------------------------------------------------- @@ -1433,6 +2279,55 @@ def test_extract_json_extends_resolved(): assert extends_edges[0].get("context") == "import" +def test_extract_json_import_and_extends_targets_are_real_nodes(tmp_path): + package_json = tmp_path / "package.json" + package_json.write_text(json.dumps({ + "name": "demo", + "dependencies": {"left-pad": "^1.3.0"}, + "devDependencies": {"bats": "^1.11.0"}, + })) + tsconfig = tmp_path / "tsconfig.json" + tsconfig.write_text(json.dumps({ + "extends": "./tsconfig.base.json", + "compilerOptions": {"strict": True}, + })) + + results = [extract_json(package_json), extract_json(tsconfig)] + combined = { + "nodes": [node for result in results for node in result["nodes"]], + "edges": [edge for result in results for edge in result["edges"]], + } + node_ids = {node["id"] for node in combined["nodes"]} + dangling = [ + edge for edge in combined["edges"] + if edge["source"] not in node_ids or edge["target"] not in node_ids + ] + assert dangling == [] + assert {"left-pad", "bats", "./tsconfig.base.json"} <= { + node["label"] for node in combined["nodes"] if node["file_type"] == "concept" + } + + extracted = extract([package_json, tsconfig], cache_root=tmp_path, parallel=False) + graph = build_from_json(extracted, directed=True) + import_targets = { + graph.nodes[data["_tgt"]]["label"] + for _, _, data in graph.edges(data=True) + if data.get("relation") == "imports" + } + extends_targets = { + graph.nodes[data["_tgt"]]["label"] + for _, _, data in graph.edges(data=True) + if data.get("relation") == "extends" + } + self_loops = [ + data for _, _, data in graph.edges(data=True) + if data.get("relation") in {"imports", "extends"} and data["_src"] == data["_tgt"] + ] + assert self_loops == [] + assert {"left-pad", "bats"} <= import_targets + assert extends_targets == {"./tsconfig.base.json"} + + def test_extract_json_large_file_skipped(tmp_path): big = tmp_path / "big.json" # Write a JSON file just over 1 MiB @@ -1769,3 +2664,169 @@ def test_case_insensitive_suffix_filtering(tmp_path): assert "myJSFunction()" in labels assert "MyTSClass" in labels + + +def test_extract_warns_on_code_files_with_no_ast_extractor(tmp_path, capsys): + # #1689: .r/.R is in CODE_EXTENSIONS (counted as code) but has no AST extractor, + # so R files silently contribute nothing. extract() must surface that instead of + # reporting success as if the language were mapped. + r1 = tmp_path / "analysis.R"; r1.write_text("f <- function(x) x + 1\n") + r2 = tmp_path / "helper.r"; r2.write_text("g <- function(y) y * 2\n") + py = tmp_path / "main.py"; py.write_text("def main():\n return 1\n") + + result = extract([r1, r2, py], cache_root=tmp_path) + err = capsys.readouterr().err + + assert "no AST extractor" in err + assert ".r (2)" in err # both R files grouped under the lowercased ext + assert "#1689" in err + # the Python file still extracts normally + labels = [n.get("label") for n in result["nodes"]] + assert any(str(l).startswith("main") for l in labels) + + +def test_extract_no_warning_when_all_code_has_extractors(tmp_path, capsys): + py = tmp_path / "a.py"; py.write_text("def a():\n return 1\n") + extract([py], cache_root=tmp_path) + err = capsys.readouterr().err + assert "no AST extractor" not in err + + +def test_extract_warns_when_sql_extra_missing(tmp_path, capsys, monkeypatch): + # #1745: .sql HAS a dispatch entry, so the #1689 warning can't fire, and + # extract_sql returns an "error" result when tree-sitter-sql is absent, so + # the #1666 warning skips it too. The files must not vanish silently: + # extract() surfaces them with the [sql] extra named. + monkeypatch.setitem(sys.modules, "tree_sitter_sql", None) # import -> ImportError + s1 = tmp_path / "schema.sql"; s1.write_text("CREATE TABLE users (id INT);\n") + s2 = tmp_path / "views.sql"; s2.write_text("CREATE VIEW v AS SELECT * FROM users;\n") + py = tmp_path / "main.py"; py.write_text("def main():\n return 1\n") + + result = extract([s1, s2, py], cache_root=tmp_path) + err = capsys.readouterr().err + + assert "2 .sql file(s)" in err + assert "tree_sitter_sql not installed" in err + assert 'graphifyy[sql]' in err + assert "#1745" in err + # the Python file still extracts normally + labels = [n.get("label") for n in result["nodes"]] + assert any(str(l).startswith("main") for l in labels) + + +def test_extract_no_missing_dep_warning_when_sql_installed(tmp_path, capsys): + pytest.importorskip("tree_sitter_sql") + s = tmp_path / "schema.sql"; s.write_text("CREATE TABLE users (id INT);\n") + extract([s], cache_root=tmp_path) + err = capsys.readouterr().err + assert "#1745" not in err + + +def test_extract_progress_final_line_uses_consistent_denominator(tmp_path, capsys): + # #1693: intermediate progress lines count against uncached_work; the final + # "100%" line must NOT switch to total_files (which includes cached hits and + # files with no extractor), or the count appears to jump upward at the end. + for i in range(100): + (tmp_path / f"m{i}.py").write_text(f"def f{i}():\n return {i}\n") + for i in range(5): + (tmp_path / f"s{i}.r").write_text(f"g{i} <- function(x) x\n") # no extractor + paths = sorted(tmp_path.glob("*.py")) + sorted(tmp_path.glob("*.r")) # total 105 + + extract(paths, cache_root=tmp_path, parallel=False) + out = capsys.readouterr().out + + # final progress line reports the uncached count (100), not the total (105) + assert "100/100 uncached files (100%)" in out + assert "105/105 files" not in out, "final line must not switch to total_files (#1693)" + + +def test_get_extractor_routes_matlab_m_away_from_objc(tmp_path): + # #1702: .m is shared by Objective-C and MATLAB. A real ObjC .m still routes to + # extract_objc, but a MATLAB .m must NOT be force-parsed by the ObjC grammar + # (which produces garbage) — it gets no extractor instead. + from graphify.extract import _get_extractor, extract_objc + + objc = tmp_path / "Foo.m" + objc.write_text('#import "Foo.h"\n@implementation Foo\n- (void)bar {}\n@end\n') + matlab_fn = tmp_path / "solver.m" + matlab_fn.write_text("function y = solver(x)\n y = x + 1;\nend\n") + matlab_cls = tmp_path / "Model.m" + matlab_cls.write_text("classdef Model\n methods\n function run(obj); end\n end\nend\n") + mm = tmp_path / "x.mm" + mm.write_text("#import \n@implementation X\n@end\n") + + assert _get_extractor(objc) is extract_objc # real ObjC .m -> objc + assert _get_extractor(matlab_fn) is None # MATLAB function -> no garbage + assert _get_extractor(matlab_cls) is None # MATLAB classdef -> no garbage + assert _get_extractor(mm) is extract_objc # .mm is unambiguously ObjC++ + + +def test_matlab_m_not_extracted_as_garbage(tmp_path, capsys): + # End to end: a MATLAB .m produces no (garbage) nodes and is surfaced by the + # no-AST-extractor warning (#1702 + #1689), rather than mis-parsed as ObjC. + m = tmp_path / "controller.m" + m.write_text("function u = controller(x)\n u = -x;\nend\n") + result = extract([m], cache_root=tmp_path) + assert result["nodes"] == [] # no garbage ObjC nodes + assert "no AST extractor" in capsys.readouterr().err # surfaced, not silent + + +def test_rewire_binds_cross_module_function_reference_to_definition(): + """#1781: a cross-module reference to a function must land on the real + definition, not a sourceless name-only stub (functions were excluded as + rewire targets).""" + from graphify.extract import _rewire_unique_stub_nodes + nodes = [ + {"id": "pkg_dep_get_db", "label": "get_db()", "file_type": "code", + "source_file": "pkg/dep.py", "source_location": "L1"}, + {"id": "get_db", "label": "get_db()", "file_type": "code", "source_file": ""}, + ] + edges = [{"source": "pkg_ep_route", "target": "get_db", "relation": "references", + "source_file": "pkg/ep.py", "weight": 1.0}] + _rewire_unique_stub_nodes(nodes, edges) + assert edges[0]["target"] == "pkg_dep_get_db" + assert "get_db" not in {n["id"] for n in nodes} # stub dropped + + +def test_rewire_does_not_bind_function_reference_across_language(): + """#1781 safety: a Python reference stub must not bind to a unique Go + function of the same name (mirrors the #1749 interop guard).""" + from graphify.extract import _rewire_unique_stub_nodes + nodes = [ + {"id": "svc_get_db", "label": "get_db()", "file_type": "code", + "source_file": "svc/main.go", "source_location": "L1"}, + {"id": "get_db", "label": "get_db()", "file_type": "code", "source_file": ""}, + ] + edges = [{"source": "app_route", "target": "get_db", "relation": "references", + "source_file": "app/route.py", "weight": 1.0}] + _rewire_unique_stub_nodes(nodes, edges) + assert edges[0]["target"] == "get_db" # unchanged — cross-language blocked + + +def test_rewire_does_not_bind_ambiguous_function_reference(): + """#1781 safety: two same-named functions leave the reference on the stub.""" + from graphify.extract import _rewire_unique_stub_nodes + nodes = [ + {"id": "a_get_db", "label": "get_db()", "file_type": "code", "source_file": "a.py", "source_location": "L1"}, + {"id": "b_get_db", "label": "get_db()", "file_type": "code", "source_file": "b.py", "source_location": "L1"}, + {"id": "get_db", "label": "get_db()", "file_type": "code", "source_file": ""}, + ] + edges = [{"source": "c_route", "target": "get_db", "relation": "references", + "source_file": "c.py", "weight": 1.0}] + _rewire_unique_stub_nodes(nodes, edges) + assert edges[0]["target"] == "get_db" # ambiguous — not merged + + +def test_rewire_does_not_bind_supertype_stub_to_function(): + """#1781 safety: a stub used as a base type must never resolve to a + same-named, same-language function.""" + from graphify.extract import _rewire_unique_stub_nodes + nodes = [ + {"id": "factory_BookStore", "label": "BookStore()", "file_type": "code", + "source_file": "factory.py", "source_location": "L1"}, + {"id": "BookStore", "label": "BookStore", "file_type": "code", "source_file": ""}, + ] + edges = [{"source": "store_Sqlite", "target": "BookStore", "relation": "inherits", + "source_file": "store.py", "weight": 1.0}] + _rewire_unique_stub_nodes(nodes, edges) + assert edges[0]["target"] == "BookStore" # inherits stub not bound to function diff --git a/tests/test_extract_cache_location.py b/tests/test_extract_cache_location.py new file mode 100644 index 000000000..89acc9c0c --- /dev/null +++ b/tests/test_extract_cache_location.py @@ -0,0 +1,141 @@ +"""#1774 — extract() must never write its AST cache into the analyzed source tree. + +The cache is an output. When no cache_root is given it used to default to the +inferred common parent of the input files — the source tree — so analyzing a +read-only corpus (someone else's repo, a knowledge base) silently created +graphify-out/cache/ inside it. It now defaults to the current working directory; +an explicit cache_root still wins. + +Crucially, the cache *location* is decoupled from the key/id *anchor*: the +inferred common parent still anchors the content-hash keys, node ids and the +XAML scan boundary, so keys stay relative and portable even when the corpus +lives outside CWD. (An earlier one-line fix that pointed the anchor itself at +CWD would have made keys absolute and machine-specific for out-of-CWD corpora.) +""" +from __future__ import annotations + +from pathlib import Path + +import graphify.extract as ex +import graphify.cache as cache +from graphify.cache import load_cached, file_hash + + +def _reset_stat_index(): + """The stat-index location is chosen once per process via a module global + (#1747). Reset it so a test sees a fresh-process decision — otherwise an + earlier test pins the location and masks where THIS extract would write it.""" + cache._stat_index_root = None + cache._stat_index = {} + cache._stat_index_dirty = False + + +def _make_corpus(base: Path) -> Path: + corpus = base / "corpus" + corpus.mkdir() + (corpus / "a.py").write_text("class Base:\n def hello(self):\n return 1\n") + (corpus / "b.py").write_text("from a import Base\n\nclass Sub(Base):\n pass\n") + return corpus + + +def test_default_cache_lands_in_cwd_not_source_tree(tmp_path, monkeypatch): + _reset_stat_index() + corpus = _make_corpus(tmp_path) + work = tmp_path / "work" + work.mkdir() + monkeypatch.chdir(work) + + result = ex.extract([corpus / "a.py", corpus / "b.py"], parallel=False) + + assert result["nodes"], "extraction should still produce nodes" + # Nothing at all in the source tree — not the AST cache, and not the + # stat-index.json the hash fastpath writes (which file_hash used to anchor on + # the key-root, leaving a stray graphify-out/ in a writable corpus, #1774). + assert not (corpus / "graphify-out").exists(), ( + "cache/stat-index written into the analyzed source tree (#1774)" + ) + assert (work / "graphify-out" / "cache").is_dir(), "cache should land under CWD" + + +def test_default_cache_does_not_leave_stat_index_in_source_tree(tmp_path, monkeypatch): + """Fresh-process regression for the stat-index leak specifically: even for a + WRITABLE out-of-CWD corpus (where the write would succeed), file_hash's + stat-index must follow the cache location, not the key anchor (#1774).""" + _reset_stat_index() + corpus = _make_corpus(tmp_path) + work = tmp_path / "elsewhere" + work.mkdir() + monkeypatch.chdir(work) + + ex.extract([corpus / "a.py", corpus / "b.py"], parallel=False) + # The stat index is buffered in memory and flushed at interpreter exit; force + # the flush now so we can assert WHERE it lands. + cache._flush_stat_index() + + assert not (corpus / "graphify-out").exists(), "stat-index leaked into the corpus" + assert (work / "graphify-out" / "cache" / "stat-index.json").exists(), ( + "stat-index should be written under the cache location (CWD)" + ) + + +def test_explicit_cache_root_still_wins(tmp_path, monkeypatch): + _reset_stat_index() + corpus = _make_corpus(tmp_path) + work = tmp_path / "work" + work.mkdir() + out = tmp_path / "out" + monkeypatch.chdir(work) + + ex.extract([corpus / "a.py"], cache_root=out, parallel=False) + + assert (out / "graphify-out" / "cache").is_dir() + assert not (corpus / "graphify-out").exists() + assert not (work / "graphify-out").exists() + + +def test_default_cache_round_trips_via_extract(tmp_path, monkeypatch): + """A second extract() of the same corpus must hit the CWD cache the first + wrote — the real contract (both runs anchor keys on the inferred corpus root + and locate the cache at CWD).""" + corpus = _make_corpus(tmp_path) + work = tmp_path / "work" + work.mkdir() + monkeypatch.chdir(work) + + ex.extract([corpus / "a.py"], parallel=False) + # Look up with the same anchor extract() uses (the corpus dir) and the CWD + # cache location — this must hit. + hit = load_cached(corpus / "a.py", corpus.resolve(), cache_root=Path(".").resolve()) + assert hit is not None, "second run should hit the CWD cache written by the first" + + +def test_cache_keys_stay_relative_for_out_of_cwd_corpus(tmp_path, monkeypatch): + """The location/anchor split must keep content-hash keys anchored on the + corpus (relative, portable) even when the corpus is outside CWD — not + re-anchored to CWD, which would bake a machine-specific absolute path into + the key and break shared/CI cache reuse (#1774 regression guard).""" + import hashlib + + corpus = _make_corpus(tmp_path) + work = tmp_path / "elsewhere" / "work" + work.mkdir(parents=True) + monkeypatch.chdir(work) + + ex.extract([corpus / "a.py"], parallel=False) + + root = corpus.resolve() + key = file_hash(corpus / "a.py", root) + raw = (corpus / "a.py").read_bytes() + + def _key_with(anchor_rel: str) -> str: + h = hashlib.sha256() + h.update(raw) + h.update(b"\x00") + h.update(anchor_rel.encode()) + return h.hexdigest() + + # Portable: keyed on the relative path within the corpus... + assert key == _key_with("a.py") + # ...not on the absolute path (which the CWD-anchor one-liner would produce). + abs_rel = str((corpus / "a.py").resolve()).lower() + assert key != _key_with(abs_rel) diff --git a/tests/test_extract_cli.py b/tests/test_extract_cli.py index c301c50e5..b1c369a5e 100644 --- a/tests/test_extract_cli.py +++ b/tests/test_extract_cli.py @@ -1,6 +1,8 @@ """Tests for `graphify extract` CLI dispatch path in graphify.__main__.""" from __future__ import annotations +import os + import pytest import graphify.__main__ as mainmod @@ -102,6 +104,15 @@ def _one_chunk_succeeded(paths, **kwargs): monkeypatch.setattr( "graphify.llm.extract_corpus_parallel", _one_chunk_succeeded ) + cache_call = {} + + def _capture_semantic_cache(*args, **kwargs): + cache_call.update(kwargs) + return 0 + + monkeypatch.setattr( + "graphify.cache.save_semantic_cache", _capture_semantic_cache + ) monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) monkeypatch.setattr( mainmod.sys, @@ -121,6 +132,450 @@ def _one_chunk_succeeded(paths, **kwargs): assert (out_dir / "graphify-out" / "graph.json").exists(), ( "graph.json must be written on the happy path" ) + assert { + str(path) for path in cache_call["allowed_source_files"] + } == {str(corpus / "README.md")} + + +def test_incremental_partial_run_preserves_untouched_semantic_hash( + monkeypatch, tmp_path +): + """#1948 caller-side guard: an incremental run that only re-dispatches the + CHANGED subset must not blank semantic_hash for live-but-untouched files. + + clear_semantic must be derived from what was actually SENT to the backend + this run (semantic_files), not from the full live corpus (files_by_type): + with the latter, every unchanged doc lands in the clear set on every + incremental run, so the very next run re-extracts the whole corpus, + forever.""" + import json + + corpus = _make_corpus(tmp_path) # main.go + README.md + (corpus / "OTHER.md").write_text("# Other\nAn independent second doc.\n") + out_dir = tmp_path / "out" + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") + + dispatched: list[list[str]] = [] + + def _stamp_everything_sent(paths, **kwargs): + sent = sorted(os.path.relpath(str(p), str(corpus)) for p in paths) + dispatched.append(sent) + on_chunk = kwargs.get("on_chunk_done") + if on_chunk: + on_chunk(0, 1, {"nodes": [], "edges": [], "hyperedges": []}) + return { + "nodes": [{"id": f"n-{rel}", "source_file": rel, + "file_type": "document"} for rel in sent], + "edges": [], + "hyperedges": [], + "input_tokens": 10, + "output_tokens": 5, + } + + monkeypatch.setattr( + "graphify.llm.extract_corpus_parallel", _stamp_everything_sent + ) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + def _run_extract(): + monkeypatch.setattr( + mainmod.sys, "argv", + ["graphify", "extract", str(corpus), "--backend", "claude", + "--no-cluster", "--out", str(out_dir)], + ) + try: + mainmod.main() + except SystemExit as exc: + assert exc.code in (None, 0), f"unexpected exit code {exc.code}" + + # Run 1: full scan — both docs dispatched and stamped. + _run_extract() + manifest_path = out_dir / "graphify-out" / "manifest.json" + m1 = json.loads(manifest_path.read_text()) + assert m1["README.md"].get("semantic_hash") + assert m1["OTHER.md"].get("semantic_hash") + + # Run 2: only README.md changes → the incremental gate dispatches it alone. + (corpus / "README.md").write_text("# Notes\nChanged content, new hash.\n") + _run_extract() + assert dispatched[-1] == ["README.md"], ( + f"run 2 should dispatch only the changed doc, got {dispatched[-1]}" + ) + m2 = json.loads(manifest_path.read_text()) + assert m2["README.md"].get("semantic_hash") + # The heart of the guard: an untouched, never-dispatched live doc keeps + # its stamp across a partial incremental run. + assert m2["OTHER.md"].get("semantic_hash"), ( + "untouched doc's semantic_hash was blanked by a partial incremental " + "run — clear_semantic was derived from the full live corpus instead " + "of the dispatched subset (#1948)" + ) + + +def test_truncated_doc_semantic_hash_is_cleared_for_requeue(monkeypatch, tmp_path): + """#1948 x #1950 interaction: a doc stamped complete on a prior run that + TRUNCATES (partial) this run must have its stale semantic_hash cleared, so + detect_incremental re-queues it — not inherit the old hash and look + unchanged. Partial files are dropped by _stamped_manifest_files, so they + land in clear_semantic (dispatched-but-not-stamped).""" + import json + + corpus = _make_corpus(tmp_path) # main.go + README.md + out_dir = tmp_path / "out" + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") + partial_run = {"on": False} + + def _extract(paths, **kwargs): + rels = sorted(os.path.relpath(str(p), str(corpus)) for p in paths) + on_chunk = kwargs.get("on_chunk_done") + if on_chunk: + on_chunk(0, 1, {"nodes": [], "edges": [], "hyperedges": []}) + node = {"id": "n-readme", "source_file": "README.md", "file_type": "document"} + if partial_run["on"] and "README.md" in rels: + node["_partial"] = True # this run truncated README.md + return {"nodes": [node] if "README.md" in rels else [], + "edges": [], "hyperedges": [], "input_tokens": 10, "output_tokens": 5} + + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _extract) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + def _run(): + monkeypatch.setattr(mainmod.sys, "argv", + ["graphify", "extract", str(corpus), "--backend", "claude", + "--no-cluster", "--out", str(out_dir)]) + try: + mainmod.main() + except SystemExit as exc: + assert exc.code in (None, 0) + + manifest_path = out_dir / "graphify-out" / "manifest.json" + _run() # run 1: complete + assert json.loads(manifest_path.read_text())["README.md"].get("semantic_hash") + + # run 2: README.md changes and truncates (partial) this time. + (corpus / "README.md").write_text("# Notes\nNew, longer content that truncated.\n") + partial_run["on"] = True + _run() + m2 = json.loads(manifest_path.read_text()) + assert not m2.get("README.md", {}).get("semantic_hash"), ( + "a truncated doc's stale semantic_hash must be cleared so it is " + "re-queued next run (#1948 x #1950)" + ) + + +def test_manifest_stamps_freshly_extracted_semantic_docs(monkeypatch, tmp_path): + """#1897: fresh extraction returns nodes with ROOT-RELATIVE source_file, + while the #933 manifest filter compared them against detect()'s ABSOLUTE + paths — so `f in _sem_extracted` was always False and every freshly + extracted doc was dropped from the manifest (only code/zero-node files + survived). Both sides must be resolved against the scan root; a genuinely + omitted doc (zero nodes) must still stay unstamped (#933 is intentional).""" + import json + + corpus = _make_corpus(tmp_path) # main.go + README.md + (corpus / "OMITTED.md").write_text("# never extracted\n") + out_dir = tmp_path / "out" + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") + + def _fresh_relative(paths, **kwargs): + on_chunk = kwargs.get("on_chunk_done") + if on_chunk: + on_chunk(0, 1, {"nodes": [], "edges": [], "hyperedges": []}) + # Root-relative source_file, exactly what a fresh extraction produces. + # OMITTED.md gets no nodes/edges — the model skipped it. + return { + "nodes": [{"id": "readme", "source_file": "README.md", + "file_type": "document"}], + "edges": [], + "hyperedges": [], + "input_tokens": 10, + "output_tokens": 5, + } + + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _fresh_relative) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, "argv", + ["graphify", "extract", str(corpus), "--backend", "claude", + "--no-cluster", "--out", str(out_dir)], + ) + + try: + mainmod.main() + except SystemExit as exc: + assert exc.code in (None, 0), f"unexpected exit code {exc.code}" + + manifest_path = out_dir / "graphify-out" / "manifest.json" + assert manifest_path.exists() + manifest = json.loads(manifest_path.read_text()) + + assert "README.md" in manifest, ( + f"freshly-extracted doc missing from manifest (#1897): {sorted(manifest)}" + ) + assert manifest["README.md"].get("semantic_hash"), ( + "freshly-extracted doc must carry a non-empty semantic_hash" + ) + # Code files are always stamped. + assert manifest.get("main.go", {}).get("semantic_hash") + # The zero-node doc stays unstamped so detect_incremental re-queues it (#933). + assert "OMITTED.md" not in manifest, ( + "zero-node doc must not be stamped in the manifest" + ) + + +def test_stamped_manifest_files_normalizes_both_sides(tmp_path): + """Unit test for the #1897 helper: relative (fresh) and absolute (cache-hit) + source_file values must both match detect()'s absolute file lists; docs with + no output are filtered; code files pass through untouched.""" + from graphify.cli import _stamped_manifest_files + + fresh_doc = tmp_path / "fresh.md"; fresh_doc.write_text("# fresh") + cached_doc = tmp_path / "cached.md"; cached_doc.write_text("# cached") + omitted_doc = tmp_path / "omitted.md"; omitted_doc.write_text("# omitted") + code = tmp_path / "app.py"; code.write_text("x = 1") + + files_by_type = { + "code": [str(code)], + "document": [str(fresh_doc), str(cached_doc), str(omitted_doc)], + } + sem_result = { + # fresh extraction: root-relative source_file + "nodes": [{"id": "n1", "source_file": "fresh.md"}], + # cache replay: absolute source_file (edge-only coverage counts too) + "edges": [{"source": "a", "target": "b", "source_file": str(cached_doc)}], + } + + out = _stamped_manifest_files(files_by_type, sem_result, tmp_path) + assert out["code"] == [str(code)] + assert out["document"] == [str(fresh_doc), str(cached_doc)] + + +def test_stamped_manifest_files_counts_hyperedge_only_docs(tmp_path): + """#1920: a doc whose only chunk output is a hyperedge (3+ nodes sharing a + concept) is valid output — the semantic cache persists it per source_file — + so it must be stamped. Before the fix the stamping loop only inspected + ``nodes``/``edges``, leaving such a doc unstamped and re-queued forever.""" + from graphify.cli import _stamped_manifest_files + + hyper_doc = tmp_path / "hyper.md"; hyper_doc.write_text("# hyper") + omitted_doc = tmp_path / "omitted.md"; omitted_doc.write_text("# omitted") + + files_by_type = {"document": [str(hyper_doc), str(omitted_doc)]} + sem_result = { + "nodes": [], + "edges": [], + "hyperedges": [ + {"id": "h1", "label": "L", "nodes": ["a", "b", "c"], + "relation": "participate_in", "source_file": "hyper.md"}, + ], + } + + out = _stamped_manifest_files(files_by_type, sem_result, tmp_path) + assert str(hyper_doc) in out["document"], ( + "a hyperedge-only doc must be stamped (#1920)" + ) + # A doc with no output at all still stays unstamped (#933). + assert str(omitted_doc) not in out["document"] + + +def test_manifest_stamps_hyperedge_only_docs(monkeypatch, tmp_path): + """#1920 end-to-end: a fresh extraction whose only output for a doc is a + hyperedge stamps that doc's semantic_hash, so it is not re-dispatched.""" + import json + + corpus = _make_corpus(tmp_path) # main.go + README.md + out_dir = tmp_path / "out" + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") + + def _hyperedge_only(paths, **kwargs): + on_chunk = kwargs.get("on_chunk_done") + if on_chunk: + on_chunk(0, 1, {"nodes": [], "edges": [], "hyperedges": []}) + return { + "nodes": [], + "edges": [], + "hyperedges": [{"id": "h1", "label": "Shared", "nodes": ["a", "b", "c"], + "relation": "participate_in", "source_file": "README.md"}], + "input_tokens": 10, + "output_tokens": 5, + } + + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _hyperedge_only) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, "argv", + ["graphify", "extract", str(corpus), "--backend", "claude", + "--no-cluster", "--out", str(out_dir)], + ) + try: + mainmod.main() + except SystemExit as exc: + assert exc.code in (None, 0), f"unexpected exit code {exc.code}" + + manifest = json.loads((out_dir / "graphify-out" / "manifest.json").read_text()) + assert manifest.get("README.md", {}).get("semantic_hash"), ( + f"hyperedge-only doc must be stamped (#1920): {sorted(manifest)}" + ) + + +# --- #1894: --force and deep-mode dispatch over a warm cache ----------------- + +def _recording_extractor(calls): + """extract_corpus_parallel stand-in that records each dispatch.""" + def _extract(paths, **kwargs): + calls.append({"paths": [str(p) for p in paths], "kwargs": kwargs}) + on_chunk = kwargs.get("on_chunk_done") + if on_chunk: + on_chunk(0, 1, {"nodes": [], "edges": [], "hyperedges": []}) + return { + "nodes": [{"id": "readme", "source_file": "README.md", + "file_type": "document"}], + "edges": [], + "hyperedges": [], + "input_tokens": 10, + "output_tokens": 5, + } + return _extract + + +def _run_extract(monkeypatch, argv): + monkeypatch.setattr(mainmod.sys, "argv", argv) + try: + mainmod.main() + except SystemExit as exc: + assert exc.code in (None, 0), f"unexpected exit code {exc.code}" + + +def test_extract_mode_deep_dispatches_over_warm_cache(monkeypatch, tmp_path): + """#1894 repro: over a warm manifest + warm standard semantic cache, + `extract --mode deep` was a silent no-op — the incremental gate dispatched + zero files before the cache was ever consulted, and the cache key ignored + mode anyway. Deep must re-dispatch on the first deep run (deep namespace + cold) and be served from cache/semantic-deep/ on the second.""" + corpus = _make_corpus(tmp_path) + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") + monkeypatch.delenv("GRAPHIFY_FORCE", raising=False) + calls: list[dict] = [] + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", + _recording_extractor(calls)) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + # No --out: the default layout (graphify-out/ beside the sources) keeps the + # CLI-level cache write's root anchored at the corpus, so the stub's + # root-relative source_file resolves (real runs also checkpoint per chunk + # inside llm.extract_corpus_parallel, which this stub replaces). + base = ["graphify", "extract", str(corpus), "--backend", "claude", + "--no-cluster"] + + # Run 1: cold standard extraction — warms manifest + plain semantic cache. + _run_extract(monkeypatch, base) + assert len(calls) == 1 + + # Sanity: a warm standard re-run dispatches nothing (expected behavior). + _run_extract(monkeypatch, base) + assert len(calls) == 1 + + # The repro: warm tree + --mode deep MUST dispatch. + _run_extract(monkeypatch, base + ["--mode", "deep"]) + assert len(calls) == 2, ( + "--mode deep over a warm cache must re-dispatch (#1894)" + ) + assert calls[1]["paths"] == [str(corpus / "README.md")] + assert calls[1]["kwargs"].get("deep_mode") is True + + # Second deep run: served from the (now warm) deep namespace, no dispatch. + _run_extract(monkeypatch, base + ["--mode", "deep"]) + assert len(calls) == 2, ( + "second deep run must be served from cache/semantic-deep/" + ) + # The deep entry landed in its own namespace, not cache/semantic/. Entries are + # nested under a p{prompt-fingerprint}/ subdir (#1939), hence the recursive glob. + assert any((corpus / "graphify-out" / "cache" / "semantic-deep").glob("**/*.json")) + + +def test_extract_force_flag_redispatches_and_stamps_manifest(monkeypatch, tmp_path): + """extract accepts --force: a warm tree re-dispatches every semantic file + (cache read skipped, incremental gate off) and the manifest is still + stamped afterward (#1897-compatible full coverage).""" + import json + + corpus = _make_corpus(tmp_path) + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") + monkeypatch.delenv("GRAPHIFY_FORCE", raising=False) + calls: list[dict] = [] + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", + _recording_extractor(calls)) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + base = ["graphify", "extract", str(corpus), "--backend", "claude", + "--no-cluster"] + + _run_extract(monkeypatch, base) + assert len(calls) == 1 + _run_extract(monkeypatch, base) # warm: no dispatch + assert len(calls) == 1 + + _run_extract(monkeypatch, base + ["--force"]) + assert len(calls) == 2, ( + "--force over a warm tree must re-dispatch every semantic file" + ) + assert calls[1]["paths"] == [str(corpus / "README.md")] + + # The forced run still wrote the semantic cache and stamped the manifest. + # Entries nest under a p{prompt-fingerprint}/ subdir (#1939). + assert any((corpus / "graphify-out" / "cache" / "semantic").glob("**/*.json")) + manifest = json.loads( + (corpus / "graphify-out" / "manifest.json").read_text() + ) + assert manifest.get("README.md", {}).get("semantic_hash"), ( + "forced re-dispatch must still stamp the manifest" + ) + assert manifest.get("main.go", {}).get("semantic_hash") + + +def test_extract_graphify_force_env_redispatches(monkeypatch, tmp_path): + """GRAPHIFY_FORCE=1 behaves like --force (env parity with `update`).""" + corpus = _make_corpus(tmp_path) + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") + monkeypatch.delenv("GRAPHIFY_FORCE", raising=False) + calls: list[dict] = [] + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", + _recording_extractor(calls)) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + base = ["graphify", "extract", str(corpus), "--backend", "claude", + "--no-cluster"] + + _run_extract(monkeypatch, base) + assert len(calls) == 1 + _run_extract(monkeypatch, base) # warm: no dispatch + assert len(calls) == 1 + + monkeypatch.setenv("GRAPHIFY_FORCE", "1") + _run_extract(monkeypatch, base) + assert len(calls) == 2, "GRAPHIFY_FORCE=1 must force a re-dispatch" + + +def test_cache_check_mode_deep_reads_deep_namespace(monkeypatch, tmp_path, capsys): + """cache-check --mode deep consults cache/semantic-deep/; without the flag + it keeps reading cache/semantic/ (deep entries are invisible to it).""" + from graphify.cache import save_semantic_cache + + doc = tmp_path / "doc.md" + doc.write_text("# Doc\n") + save_semantic_cache([{"id": "d", "source_file": "doc.md"}], [], + root=tmp_path, mode="deep") + files_from = tmp_path / "files.txt" + files_from.write_text(str(doc) + "\n") + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + _run_extract(monkeypatch, ["graphify", "cache-check", str(files_from), + "--root", str(tmp_path)]) + assert "Cache: 0 hit, 1 miss" in capsys.readouterr().out + + _run_extract(monkeypatch, ["graphify", "cache-check", str(files_from), + "--root", str(tmp_path), "--mode", "deep"]) + assert "Cache: 1 hit, 0 miss" in capsys.readouterr().out def _code_only_corpus(tmp_path): @@ -172,6 +627,73 @@ def test_extract_codeonly_succeeds_without_api_key(monkeypatch, tmp_path): assert len(json.loads(graph.read_text()).get("nodes", [])) > 0 +def test_missing_manifest_code_only_preserves_semantic_layer(monkeypatch, tmp_path): + """#1925: `graphify extract --code-only` with a MISSING manifest.json must + not degrade to a full scan that discards the committed semantic layer. An + existing graph.json is a sufficient incremental baseline, so doc/paper/image + nodes (excluded by --code-only, not deleted) are preserved; a genuinely + deleted source is still evicted (#1909 semantics retained).""" + import json + + corpus = tmp_path / "proj"; corpus.mkdir() + (corpus / "keep.py").write_text("def keep():\n return 1\n") + (corpus / "README.md").write_text("# Notes\nCurated docs.\n") + out_dir = tmp_path / "out" + graphify_out = out_dir / "graphify-out" + _clear_backend_keys(monkeypatch) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + def _sem_doc_count(g): + return sum(1 for n in g["nodes"] if n.get("source_file") == "README.md") + + # 1) seed a code-only graph + _run_extract(monkeypatch, ["graphify", "extract", str(corpus), + "--code-only", "--out", str(out_dir)]) + graph_path = graphify_out / "graph.json" + graph = json.loads(graph_path.read_text()) + + # 2) inject a committed semantic layer for README.md (nodes + edge + hyperedge) + graph["nodes"].append({"id": "doc_readme_a", "label": "Concept A", + "source_file": "README.md", "file_type": "document"}) + graph["nodes"].append({"id": "doc_readme_b", "label": "Concept B", + "source_file": "README.md", "file_type": "document"}) + graph.setdefault("edges", []).append( + {"source": "doc_readme_a", "target": "doc_readme_b", + "relation": "relates_to", "source_file": "README.md"}) + graph.setdefault("hyperedges", []).append( + {"id": "h1", "label": "Shared", "nodes": ["doc_readme_a", "doc_readme_b"], + "relation": "participate_in", "source_file": "README.md"}) + graph_path.write_text(json.dumps(graph)) + (graphify_out / ".graphify_semantic_marker").write_text( + json.dumps({"output_tokens": 1})) + + # 3) manifest goes missing (fresh clone / deliberately untracked) + (graphify_out / "manifest.json").unlink() + + # 4) re-run the SAME code-only extract + _run_extract(monkeypatch, ["graphify", "extract", str(corpus), + "--code-only", "--out", str(out_dir)]) + after = json.loads(graph_path.read_text()) + assert _sem_doc_count(after) >= 2, ( + "committed semantic doc nodes must survive a missing-manifest " + f"--code-only rebuild (#1925); got {_sem_doc_count(after)}" + ) + assert any(h.get("id") == "h1" for h in after.get("hyperedges", [])), ( + "committed hyperedge must survive the rebuild" + ) + assert any("keep" in n["id"] for n in after["nodes"]), "code nodes intact" + + # 5) a genuine deletion still evicts the doc's semantic nodes + (corpus / "README.md").unlink() + (graphify_out / "manifest.json").unlink(missing_ok=True) + _run_extract(monkeypatch, ["graphify", "extract", str(corpus), + "--code-only", "--out", str(out_dir)]) + gone = json.loads(graph_path.read_text()) + assert _sem_doc_count(gone) == 0, ( + "a genuinely deleted doc must still be evicted (#1909 semantics preserved)" + ) + + def test_extract_out_keeps_project_root_clean(monkeypatch, tmp_path): """`extract --out DIR` routes every artifact to DIR/graphify-out/ and the scanned project must not grow a graphify-out/ (or anything else) beside @@ -265,3 +787,307 @@ def test_extract_timing_flag_emits_stage_timings(monkeypatch, tmp_path, capsys): mainmod.main() assert exc2.value.code == 0 assert "graphify timing" not in capsys.readouterr().err + + +@pytest.mark.parametrize( + "postgres_args", + [["--postgres", "test-dsn"], ["--postgres=test-dsn"]], +) +@pytest.mark.parametrize("cluster_args", [[], ["--no-cluster"]]) +def test_pathless_postgres_extract_initializes_empty_detection( + monkeypatch, tmp_path, postgres_args, cluster_args +): + calls = [] + + def _introspect(dsn): + calls.append(dsn) + return { + "nodes": [ + { + "id": "postgresql_users", + "label": "users", + "type": "table", + "file_type": "code", + "source_file": "postgresql:/localhost/test", + } + ], + "edges": [], + } + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "app.py").write_text("def app():\n return 1\n") + launcher = tmp_path / "launcher" + launcher.mkdir() + monkeypatch.chdir(launcher) + out_root = tmp_path / "output" + graph_path = out_root / "graphify-out" / "graph.json" + manifest = out_root / "graphify-out" / "manifest.json" + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr("graphify.pg_introspect.introspect_postgres", _introspect) + + def _run(argv): + monkeypatch.setattr(mainmod.sys, "argv", argv) + try: + mainmod.main() + except SystemExit as exc: + assert exc.code in (None, 0) + + _run( + [ + "graphify", + "extract", + str(corpus), + "--code-only", + "--no-cluster", + "--out", + str(out_root), + ] + ) + assert manifest.exists() + assert "app.py" in _node_sources(graph_path) + manifest_content = manifest.read_text() + (out_root / "graphify-out" / ".graphify_semantic_marker").write_text( + '{"output_tokens": 1}' + ) + + cache_entry = ( + out_root + / "graphify-out" + / "cache" + / "semantic" + / "deadbeef.json" + ) + cache_entry.parent.mkdir(parents=True) + cache_entry.write_text('{"nodes": [], "edges": []}') + _run( + [ + "graphify", + "extract", + *postgres_args, + *cluster_args, + "--out", + str(out_root), + ] + ) + assert calls == ["test-dsn"] + assert cache_entry.exists() + assert not manifest.exists() + assert "postgresql:/localhost/test" in _node_sources(graph_path) + backups = [ + path + for path in (out_root / "graphify-out").iterdir() + if path.is_dir() and (path / "manifest.json").exists() + ] + assert backups + assert backups[0].joinpath("manifest.json").read_text() == manifest_content + + _run( + [ + "graphify", + "extract", + str(corpus), + "--code-only", + "--no-cluster", + "--out", + str(out_root), + ] + ) + final_sources = _node_sources(graph_path) + assert "app.py" in final_sources + assert "postgresql:/localhost/test" not in final_sources + assert manifest.exists() + + +# --------------------------------------------------------------------------- +# #1909: a newly-excluded file's nodes must be pruned from graph.json on the +# next incremental extract even when the manifest never listed the file (the +# pre-#1897 state every 0.9.16 graph is in), so the manifest-diff prune set +# (`manifest - corpus`) can never see it. +# --------------------------------------------------------------------------- + +def _two_file_corpus(tmp_path): + project = tmp_path / "project" + project.mkdir() + (project / "x.py").write_text( + "def secret_helper():\n return 42\n\n" + "def secret_caller():\n return secret_helper()\n" + ) + (project / "keep.py").write_text( + "def kept():\n return still_here()\n\n" + "def still_here():\n return 1\n" + ) + return project + + +def _node_sources(graph_path): + import json + data = json.loads(graph_path.read_text(encoding="utf-8")) + return {n.get("source_file", "") for n in data.get("nodes", [])} + + +def _run_extract(monkeypatch, argv): + monkeypatch.setattr(mainmod.sys, "argv", argv) + try: + mainmod.main() + except SystemExit as exc: + assert exc.code in (None, 0), f"unexpected exit code {exc.code}" + + +def test_incremental_extract_prunes_newly_excluded_file_not_in_manifest( + monkeypatch, tmp_path +): + """Seed a graph with nodes for x.py, drop x.py from the manifest (pre-#1897 + manifests never listed excluded/omitted files), exclude x.py via + .graphifyignore, re-run extract: x.py's nodes must be gone even though it + was never on the deleted list.""" + import json + project = _two_file_corpus(tmp_path) + out_dir = tmp_path / "out" + _clear_backend_keys(monkeypatch) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + _run_extract( + monkeypatch, + ["graphify", "extract", str(project), "--out", str(out_dir)], + ) + graph_path = out_dir / "graphify-out" / "graph.json" + manifest_path = out_dir / "graphify-out" / "manifest.json" + assert any("x.py" in s for s in _node_sources(graph_path)), ( + "seed extract must produce nodes for x.py" + ) + + # Simulate the pre-#1897 manifest state: x.py was never manifest-listed, + # so `manifest - corpus` can never flag it. + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest = {k: v for k, v in manifest.items() if "x.py" not in k} + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + + (project / ".graphifyignore").write_text("x.py\n") + _run_extract( + monkeypatch, + ["graphify", "extract", str(project), "--out", str(out_dir)], + ) + + sources = _node_sources(graph_path) + assert not any("x.py" in s for s in sources), ( + f"newly-excluded x.py must be pruned from graph.json, still see {sources}" + ) + assert any("keep.py" in s for s in sources), ( + "unchanged keep.py nodes must survive the incremental merge" + ) + # x.py exists on disk, is excluded, and must not creep into the manifest. + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + assert not any("x.py" in k for k in manifest), ( + f"excluded x.py must not be (re)listed in the manifest: {set(manifest)}" + ) + + +def test_incremental_extract_prunes_excluded_file_listed_in_manifest( + monkeypatch, tmp_path +): + """Post-#1897 state: the excluded file IS manifest-listed. It must be + pruned from graph.json AND dropped from the manifest (#1908), and stay + settled on a further run.""" + import json + project = _two_file_corpus(tmp_path) + out_dir = tmp_path / "out" + _clear_backend_keys(monkeypatch) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + _run_extract( + monkeypatch, + ["graphify", "extract", str(project), "--out", str(out_dir)], + ) + graph_path = out_dir / "graphify-out" / "graph.json" + manifest_path = out_dir / "graphify-out" / "manifest.json" + assert any("x.py" in k for k in json.loads(manifest_path.read_text())) + + (project / ".graphifyignore").write_text("x.py\n") + _run_extract( + monkeypatch, + ["graphify", "extract", str(project), "--out", str(out_dir)], + ) + + sources = _node_sources(graph_path) + assert not any("x.py" in s for s in sources) + assert any("keep.py" in s for s in sources) + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + assert not any("x.py" in k for k in manifest), ( + "excluded-but-alive manifest row must be pruned (#1908)" + ) + + # Steady state: a third run neither resurrects x.py nor loses keep.py. + _run_extract( + monkeypatch, + ["graphify", "extract", str(project), "--out", str(out_dir)], + ) + sources = _node_sources(graph_path) + assert not any("x.py" in s for s in sources) + assert any("keep.py" in s for s in sources) + + +def test_no_cluster_incremental_prunes_newly_excluded_file( + monkeypatch, tmp_path, capsys +): + """--no-cluster's exclusion-only early exit must still scrub the excluded + file's nodes from the raw graph.json (that path never runs build_merge), + and must not report the alive file as deleted.""" + import json + project = _two_file_corpus(tmp_path) + out_dir = tmp_path / "out" + _clear_backend_keys(monkeypatch) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + monkeypatch.setattr( + mainmod.sys, "argv", + ["graphify", "extract", str(project), "--no-cluster", "--out", str(out_dir)], + ) + with pytest.raises(SystemExit) as exc: + mainmod.main() + assert exc.value.code == 0 + graph_path = out_dir / "graphify-out" / "graph.json" + assert any("x.py" in s for s in _node_sources(graph_path)) + capsys.readouterr() + + (project / ".graphifyignore").write_text("x.py\n") + with pytest.raises(SystemExit) as exc: + mainmod.main() + assert exc.value.code == 0 + out_text = capsys.readouterr().out + assert "1 deleted" not in out_text, ( + "excluded-but-alive file must not be reported as deleted" + ) + + sources = _node_sources(graph_path) + assert not any("x.py" in s for s in sources), ( + f"--no-cluster early exit must prune excluded sources, still see {sources}" + ) + assert any("keep.py" in s for s in sources) + + +def test_cache_check_prompt_file_scopes_hits_to_that_prompt(monkeypatch, tmp_path, capsys): + """#1939: cache-check --prompt-file only counts entries produced by that same + extraction prompt, so an upgraded prompt reports a miss (re-extract) rather + than replaying the older vintage.""" + from graphify.cache import save_semantic_cache + + doc = tmp_path / "doc.md" + doc.write_text("# Doc\n") + spec = tmp_path / "extraction-spec.md" + spec.write_text("PROMPT V1", encoding="utf-8") + save_semantic_cache([{"id": "d", "source_file": "doc.md"}], [], + root=tmp_path, prompt_file=str(spec)) + files_from = tmp_path / "files.txt" + files_from.write_text(str(doc) + "\n") + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + + base = ["graphify", "cache-check", str(files_from), "--root", str(tmp_path)] + _run_extract(monkeypatch, base + ["--prompt-file", str(spec)]) + assert "Cache: 1 hit, 0 miss" in capsys.readouterr().out + + # An upgrade rewrites the prompt: the entry must no longer satisfy the run. + spec.write_text("PROMPT V2 — rewritten by an upgrade", encoding="utf-8") + os.utime(spec, ns=(0, 0)) + _run_extract(monkeypatch, base + ["--prompt-file", str(spec)]) + assert "Cache: 0 hit, 1 miss" in capsys.readouterr().out diff --git a/tests/test_extract_code_only_cli.py b/tests/test_extract_code_only_cli.py new file mode 100644 index 000000000..93fec48d3 --- /dev/null +++ b/tests/test_extract_code_only_cli.py @@ -0,0 +1,260 @@ +"""`graphify extract --code-only` indexes code without an LLM key (#1734). + +A mixed repo (code + docs) with no API key configured used to hard-fail on the +doc/paper/image files. `--code-only` skips the semantic pass so the code graph +still builds, and the no-key error now points users at the flag. +""" +from __future__ import annotations + +import os +import json +import subprocess +import sys +from pathlib import Path + +PYTHON = sys.executable +_KEY_VARS = ("GEMINI_API_KEY", "GOOGLE_API_KEY", "OPENAI_API_KEY", "OPENAI_BASE_URL", + "ANTHROPIC_API_KEY", "MOONSHOT_API_KEY", "DEEPSEEK_API_KEY") + + +def _mixed_repo(tmp_path: Path) -> Path: + repo = tmp_path / "repo" + repo.mkdir() + (repo / "app.py").write_text("def hello():\n return 1\n") + (repo / "README.md").write_text("# Design\n\nHow it works.\n") + (repo / "NOTES.txt").write_text("Architecture notes and rationale.\n") + return repo + + +def _run(repo: Path, *extra: str): + env = {k: v for k, v in os.environ.items() if k not in _KEY_VARS} + env["GRAPHIFY_OUT"] = str(repo / "graphify-out") + return subprocess.run( + [PYTHON, "-m", "graphify", "extract", ".", *extra], + cwd=repo, capture_output=True, text=True, env=env, + ) + + +def test_code_only_succeeds_without_key(tmp_path): + repo = _mixed_repo(tmp_path) + r = _run(repo, "--code-only") + assert r.returncode == 0, f"--code-only should succeed with no key: {r.stderr}" + out = r.stdout + r.stderr + assert "--code-only: skipping" in out + graph = repo / "graphify-out" / "graph.json" + assert graph.exists(), "code graph must still be written" + import json + g = json.loads(graph.read_text()) + labels = [n.get("label") for n in g["nodes"]] + assert any(str(l).startswith("hello") for l in labels), "code was indexed" + + +def test_mixed_repo_without_key_errors_and_points_at_code_only(tmp_path): + repo = _mixed_repo(tmp_path) + r = _run(repo) # no --code-only, no key + assert r.returncode != 0, "mixed repo with no key should still error without the flag" + assert "--code-only" in r.stderr, "the no-key error must point users at --code-only" + + +def test_extract_usage_advertises_code_only(tmp_path): + """#2071: --code-only must be discoverable in the extract usage text, not only + by triggering the no-key error. `graphify extract` with no path prints usage.""" + r = subprocess.run( + [PYTHON, "-m", "graphify", "extract"], + cwd=tmp_path, capture_output=True, text=True, + ) + assert r.returncode != 0 + assert "--code-only" in r.stdout + r.stderr, ( + "extract usage must advertise --code-only (#2071)" + ) + + +def _run_relative_out(repo: Path, *extra: str): + """Like _run but with a RELATIVE GRAPHIFY_OUT so --out/--output controls the + parent dir (an absolute GRAPHIFY_OUT would override the flag).""" + env = {k: v for k, v in os.environ.items() if k not in _KEY_VARS} + env["GRAPHIFY_OUT"] = "graphify-out" + return subprocess.run( + [PYTHON, "-m", "graphify", "extract", ".", *extra], + cwd=repo, capture_output=True, text=True, env=env, + ) + + +def test_output_flag_is_alias_of_out(tmp_path): + """#2004 part 3: `--output DIR` was silently ignored on extract (output went + to the default `/graphify-out/`). It is now an alias of `--out`.""" + repo = tmp_path / "repo" + repo.mkdir() + (repo / "app.py").write_text("def hello():\n return 1\n") + custom = tmp_path / "elsewhere" + + r = _run_relative_out(repo, "--code-only", "--no-cluster", "--output", str(custom)) + assert r.returncode == 0, r.stderr + assert (custom / "graphify-out" / "graph.json").exists(), "--output was ignored (#2004)" + assert not (repo / "graphify-out").exists(), "output must not go to the default dir" + + +def test_output_flag_inline_form(tmp_path): + repo = tmp_path / "repo" + repo.mkdir() + (repo / "app.py").write_text("def hello():\n return 1\n") + custom = tmp_path / "out2" + r = _run_relative_out(repo, "--code-only", "--no-cluster", f"--output={custom}") + assert r.returncode == 0, r.stderr + assert (custom / "graphify-out" / "graph.json").exists() + + +def test_no_gitignore_indexes_vcs_ignored_code_but_keeps_graphifyignore(tmp_path): + repo = tmp_path / "repo" + generated = repo / "proj" / "deep" / "generated" + generated.mkdir(parents=True) + (repo / ".git" / "info").mkdir(parents=True) + (repo / ".git" / "info" / "exclude").write_text("local/\n") + (repo / "proj" / ".gitignore").write_text("generated/\n") + (repo / "proj" / ".graphifyignore").write_text("hidden/\n") + (generated / "Gen.cs").write_text("namespace N { public class Gen {} }\n") + local = repo / "local" + local.mkdir() + (local / "Local.cs").write_text("namespace N { public class Local {} }\n") + hidden = repo / "proj" / "hidden" + hidden.mkdir() + (hidden / "Hidden.cs").write_text("namespace N { public class Hidden {} }\n") + + result = _run(repo, "--no-gitignore", "--no-cluster") + + assert result.returncode == 0, result.stderr + graph = json.loads((repo / "graphify-out" / "graph.json").read_text()) + sources = {Path(str(node.get("source_file", ""))).as_posix() for node in graph["nodes"]} + assert any(source.endswith("proj/deep/generated/Gen.cs") for source in sources) + assert any(source.endswith("local/Local.cs") for source in sources) + assert not any(source.endswith("proj/hidden/Hidden.cs") for source in sources) + + +def test_no_gitignore_setting_persists_across_flagless_extract(tmp_path): + """#1971 persistence: once --no-gitignore is set, a later flag-less + `graphify extract` must NOT clobber it back to honoring .gitignore (which + would make the git-ignored code silently disappear again).""" + repo = tmp_path / "repo" + gen = repo / "generated" + gen.mkdir(parents=True) + (repo / ".gitignore").write_text("generated/\n") + (repo / "app.py").write_text("def hello():\n return 1\n") + (gen / "Gen.py").write_text("def gen():\n return 2\n") + + def _sources(): + g = json.loads((repo / "graphify-out" / "graph.json").read_text()) + return {Path(str(n.get("source_file", ""))).as_posix() for n in g["nodes"]} + + r1 = _run(repo, "--no-gitignore", "--code-only", "--no-cluster") + assert r1.returncode == 0, r1.stderr + assert any(s.endswith("generated/Gen.py") for s in _sources()) + + # A plain flag-less re-extract must keep the git-ignored file (setting persisted). + r2 = _run(repo, "--code-only", "--no-cluster") + assert r2.returncode == 0, r2.stderr + assert any(s.endswith("generated/Gen.py") for s in _sources()), ( + "flag-less re-extract clobbered the persisted --no-gitignore setting (#1971)" + ) + + +def test_exclude_setting_persists_across_flagless_extract(tmp_path): + repo = tmp_path / "repo" + vendor = repo / "vendor" + vendor.mkdir(parents=True) + (repo / "app.py").write_text("def app():\n return 1\n") + (vendor / "lib.py").write_text("def vendor():\n return 2\n") + + def _sources(): + graph = json.loads((repo / "graphify-out" / "graph.json").read_text()) + return { + Path(str(node.get("source_file", ""))).as_posix() + for node in graph["nodes"] + } + + first = _run( + repo, "--exclude", "vendor", "--code-only", "--no-cluster" + ) + assert first.returncode == 0, first.stderr + assert any(source.endswith("app.py") for source in _sources()) + assert not any(source.endswith("vendor/lib.py") for source in _sources()) + + second = _run(repo, "--code-only", "--no-cluster") + assert second.returncode == 0, second.stderr + assert any(source.endswith("app.py") for source in _sources()) + assert not any(source.endswith("vendor/lib.py") for source in _sources()) + + +def test_explicit_exclude_replaces_persisted_setting_with_custom_out(tmp_path): + repo = tmp_path / "repo" + vendor = repo / "vendor" + generated = repo / "generated" + vendor.mkdir(parents=True) + generated.mkdir() + (repo / "app.py").write_text("def app():\n return 1\n") + (vendor / "lib.py").write_text("def vendor():\n return 2\n") + (generated / "gen.py").write_text("def generated():\n return 3\n") + out_root = tmp_path / "custom-output" + + env = {key: value for key, value in os.environ.items() if key not in _KEY_VARS} + env["GRAPHIFY_OUT"] = "graphify-out" + + def _run_extract(*extra: str): + return subprocess.run( + [ + PYTHON, + "-m", + "graphify", + "extract", + ".", + "--out", + str(out_root), + "--code-only", + "--no-cluster", + *extra, + ], + cwd=repo, + capture_output=True, + text=True, + env=env, + ) + + first = _run_extract("--exclude", "vendor") + assert first.returncode == 0, first.stderr + + graph_out = out_root / "graphify-out" + def _sources(): + graph = json.loads((graph_out / "graph.json").read_text()) + return { + Path(str(node.get("source_file", ""))).as_posix() + for node in graph["nodes"] + } + + persisted = _run_extract("--force") + assert persisted.returncode == 0, persisted.stderr + assert any(source.endswith("app.py") for source in _sources()) + assert not any(source.endswith("vendor/lib.py") for source in _sources()) + assert any(source.endswith("generated/gen.py") for source in _sources()) + + replacement = _run_extract("--exclude", "generated", "--force") + assert replacement.returncode == 0, replacement.stderr + sources = _sources() + assert any(source.endswith("app.py") for source in sources) + assert any(source.endswith("vendor/lib.py") for source in sources) + assert not any(source.endswith("generated/gen.py") for source in sources) + assert json.loads((graph_out / ".graphify_build.json").read_text()) == { + "excludes": ["generated"] + } + + +def test_extract_names_skipped_sensitive_files(tmp_path): + """#2106 traceability: a file dropped by the sensitive-file filter is reported + by NAME (not just a count), so a wrongly-flagged file is visible.""" + repo = tmp_path / "repo" + repo.mkdir() + (repo / "app.py").write_text("def hello():\n return 1\n") + (repo / "github_token.txt").write_text("ghp_secretvalue\n") # real secret -> skipped + r = _run(repo, "--code-only", "--no-cluster") + assert r.returncode == 0, r.stderr + out = r.stdout + r.stderr + assert "skipped as potentially sensitive" in out + assert "github_token.txt" in out, "the skipped filename must be surfaced (#2106)" diff --git a/tests/test_extractors_registry.py b/tests/test_extractors_registry.py new file mode 100644 index 000000000..db647201f --- /dev/null +++ b/tests/test_extractors_registry.py @@ -0,0 +1,44 @@ +"""Facade / registry identity guards for the per-language extractor split (#1212). + +The ``extract.py`` decomposition (#1737) moved each language extractor into its +own ``graphify/extractors/.py`` module, kept a verbatim re-export in +``graphify.extract`` (the facade every existing importer uses), and seeded a +``graphify.extractors.LANGUAGE_EXTRACTORS`` registry. Three things must stay +true for that split to be behavior-preserving: + +- the function is importable from its new per-language module, +- ``graphify.extract`` still re-exports the SAME function object (facade + identity — a stale copy or shadowing import would silently diverge), +- ``LANGUAGE_EXTRACTORS`` maps to that same object (registry identity). + +Originally proposed by @Cekaru in #1721 as a per-language check; generalized +here to sweep the whole registry so a future move that forgets the facade +re-export (or re-exports a different object) fails loudly. +""" +from __future__ import annotations + +import graphify.extract as facade +from graphify.extractors import LANGUAGE_EXTRACTORS + + +def test_every_registry_extractor_is_reexported_from_facade(): + missing = [] + diverged = [] + for lang, fn in LANGUAGE_EXTRACTORS.items(): + name = getattr(fn, "__name__", None) + if not name or not hasattr(facade, name): + missing.append((lang, name)) + continue + if getattr(facade, name) is not fn: + diverged.append((lang, name)) + assert not missing, f"registry extractors not re-exported from graphify.extract: {missing}" + assert not diverged, f"facade object diverges from registry: {diverged}" + + +def test_terraform_migrated(): + # The concrete anchor from #1721: extract_terraform lives in its own module, + # and both the facade and the registry point at that one object. + from graphify.extractors.terraform import extract_terraform + + assert facade.extract_terraform is extract_terraform + assert LANGUAGE_EXTRACTORS["terraform"] is extract_terraform diff --git a/tests/test_file_label_disambiguation.py b/tests/test_file_label_disambiguation.py new file mode 100644 index 000000000..ccefbb5a4 --- /dev/null +++ b/tests/test_file_label_disambiguation.py @@ -0,0 +1,127 @@ +"""File-node labels are disambiguated when basenames collide (#2032). + +In directory-per-entrypoint repos (Supabase Edge Functions, Next.js pages, +Rust mod.rs, Python __init__.py) many files share a basename, so basename-only +file-node labels collide and `explain`/discovery can't tell them apart. When a +basename collides, file nodes get a shortest-unique directory-qualified label; +unique basenames are left bare. Ids/edges are never changed — only labels. +""" +from __future__ import annotations + +import networkx as nx + +from graphify.build import ( + _disambiguate_file_node_labels, + _is_file_node_label, + _shortest_unique_suffix, + disambiguate_file_labels_in_nodes, +) + + +def test_disambiguate_raw_node_list_for_no_cluster_path(): + """The extract --no-cluster path writes the merged node dicts directly + (bypassing build_from_json), so the list-based variant must relabel too.""" + nodes = [ + {"id": "po", "label": "index.ts", "file_type": "code", "source_file": "fn/process-order/index.ts"}, + {"id": "sr", "label": "index.ts", "file_type": "code", "source_file": "fn/send-receipt/index.ts"}, + {"id": "m", "label": "main.ts", "file_type": "code", "source_file": "main.ts"}, + {"id": "sym", "label": "handler", "file_type": "code", "source_file": "fn/process-order/index.ts"}, + ] + disambiguate_file_labels_in_nodes(nodes) + by_id = {n["id"]: n["label"] for n in nodes} + assert by_id["po"] == "process-order/index.ts" + assert by_id["sr"] == "send-receipt/index.ts" + assert by_id["m"] == "main.ts" + assert by_id["sym"] == "handler" + + +def test_is_file_node_label_and_suffix_helpers(): + assert _is_file_node_label("index.ts", "a/b/index.ts") + assert _is_file_node_label("b/index.ts", "a/b/index.ts") # qualified form + assert not _is_file_node_label("index", "a/b/index.ts") # a symbol, not the file + assert not _is_file_node_label("helper()", "a/b/index.ts") + all_sfs = {"supabase/functions/process-order/index.ts", + "supabase/functions/send-receipt/index.ts"} + assert _shortest_unique_suffix("supabase/functions/process-order/index.ts", all_sfs) == "process-order/index.ts" + # root-level file colliding with a nested one keeps the bare basename + assert _shortest_unique_suffix("index.ts", {"index.ts", "a/index.ts"}) == "index.ts" + + +def _file_node(nid, sf): + return (nid, {"label": sf.rsplit("/", 1)[-1], "file_type": "code", "source_file": sf}) + + +def test_colliding_file_labels_are_qualified_uniques_left_bare(): + G = nx.DiGraph() + for nid, sf in [ + ("po", "supabase/functions/process-order/index.ts"), + ("sr", "supabase/functions/send-receipt/index.ts"), + ("main", "src/main.ts"), + ]: + G.add_node(nid, **_file_node(nid, sf)[1]) + # a symbol inside one of the files must NOT be relabeled + G.add_node("sym", label="handler", file_type="code", source_file="supabase/functions/process-order/index.ts") + + _disambiguate_file_node_labels(G) + + assert G.nodes["po"]["label"] == "process-order/index.ts" + assert G.nodes["sr"]["label"] == "send-receipt/index.ts" + assert G.nodes["main"]["label"] == "main.ts", "unique basename must stay bare" + assert G.nodes["sym"]["label"] == "handler", "symbol nodes must be untouched" + + +def test_disambiguation_is_idempotent(): + G = nx.DiGraph() + G.add_node("a", label="index.ts", file_type="code", source_file="x/a/index.ts") + G.add_node("b", label="index.ts", file_type="code", source_file="x/b/index.ts") + _disambiguate_file_node_labels(G) + first = {n: G.nodes[n]["label"] for n in G} + # Re-run over already-qualified labels: must be stable (derived from path). + _disambiguate_file_node_labels(G) + assert {n: G.nodes[n]["label"] for n in G} == first + assert first == {"a": "a/index.ts", "b": "b/index.ts"} + + +def test_three_way_collision_grows_suffix_until_unique(): + G = nx.DiGraph() + G.add_node("a", label="index.ts", file_type="code", source_file="a/x/index.ts") + G.add_node("b", label="index.ts", file_type="code", source_file="b/x/index.ts") + _disambiguate_file_node_labels(G) + # basename + one dir ("x/index.ts") still collides, so grow to two dirs. + assert G.nodes["a"]["label"] == "a/x/index.ts" + assert G.nodes["b"]["label"] == "b/x/index.ts" + + +def test_end_to_end_build_and_lookup(tmp_path): + """Full pipeline: two entry-point index.ts files get distinguishable labels + and both resolve via serve._find_node.""" + from graphify.extract import extract + from graphify.build import build_from_json + from graphify.serve import _find_node + + fns = tmp_path / "supabase" / "functions" + (fns / "process-order").mkdir(parents=True) + (fns / "send-receipt").mkdir(parents=True) + (fns / "process-order" / "index.ts").write_text("export function processOrder() { return 1; }\n") + (fns / "send-receipt" / "index.ts").write_text("export function sendReceipt() { return 2; }\n") + (tmp_path / "main.ts").write_text("export function main() { return 0; }\n") + + result = extract( + [fns / "process-order" / "index.ts", fns / "send-receipt" / "index.ts", tmp_path / "main.ts"], + cache_root=tmp_path / "cache", parallel=False, + ) + G = build_from_json(result, root=str(tmp_path)) + + file_labels = { + d["label"] for _, d in G.nodes(data=True) + if _is_file_node_label(d.get("label"), d.get("source_file")) + } + assert "process-order/index.ts" in file_labels + assert "send-receipt/index.ts" in file_labels + assert "main.ts" in file_labels # unique basename stays bare + + # Discovery works for both the directory name and the qualified path. + assert _find_node(G, "process-order") + assert _find_node(G, "process-order/index.ts") + po = _find_node(G, "process-order/index.ts")[0] + assert G.nodes[po]["label"] == "process-order/index.ts" diff --git a/tests/test_gemini_hook.py b/tests/test_gemini_hook.py new file mode 100644 index 000000000..aa84117dd --- /dev/null +++ b/tests/test_gemini_hook.py @@ -0,0 +1,71 @@ +"""The Gemini CLI BeforeTool guard nudges toward the graph, shell-agnostically. + +Since #522 it runs as `graphify hook-guard gemini` (not a `python -c` one-liner +that depended on a bare `python` on PATH and embedded PowerShell-hostile +backticks). It always returns {"decision":"allow"} so a tool is never blocked, +and appends additionalContext only when a graph exists. +""" +import json +import os +import subprocess +import sys + +from graphify.__main__ import _gemini_hook + + +def _env(): + e = dict(os.environ) + e.pop("GRAPHIFY_OUT", None) + return e + + +def _run(cwd, *, graph: bool): + if graph: + (cwd / "graphify-out").mkdir(parents=True, exist_ok=True) + (cwd / "graphify-out" / "graph.json").write_text("{}", encoding="utf-8") + return subprocess.run( + [sys.executable, "-m", "graphify", "hook-guard", "gemini"], + input="", capture_output=True, text=True, cwd=cwd, env=_env(), + ) + + +def test_matcher_and_command_shape(): + h = _gemini_hook() + assert h["matcher"] == "read_file|list_directory" + cmd = h["hooks"][0]["command"] + # #522: no bare `python` dependency, no embedded quote/backtick soup. + assert "python -c" not in cmd + assert "graphify" in cmd and "hook-guard gemini" in cmd + + +def test_allows_and_nudges_with_graph(tmp_path): + out = _run(tmp_path, graph=True).stdout + payload = json.loads(out) + assert payload["decision"] == "allow" + assert "graphify query" in payload["additionalContext"] + + +def test_allows_without_nudge_when_no_graph(tmp_path): + out = _run(tmp_path, graph=False).stdout + payload = json.loads(out) + assert payload["decision"] == "allow" + assert "additionalContext" not in payload + + +def test_never_blocks(tmp_path): + r = _run(tmp_path, graph=True) + assert r.returncode == 0 + payload = json.loads(r.stdout) + assert payload["decision"] == "allow" + + +def test_honors_graphify_out_override(tmp_path): + custom = tmp_path / "custom-out" + custom.mkdir() + (custom / "graph.json").write_text("{}", encoding="utf-8") + env = dict(os.environ, GRAPHIFY_OUT=str(custom)) + r = subprocess.run( + [sys.executable, "-m", "graphify", "hook-guard", "gemini"], + input="", capture_output=True, text=True, cwd=tmp_path, env=env, + ) + assert "graphify query" in json.loads(r.stdout).get("additionalContext", "") diff --git a/tests/test_god_nodes_cli.py b/tests/test_god_nodes_cli.py new file mode 100644 index 000000000..28bacc9b1 --- /dev/null +++ b/tests/test_god_nodes_cli.py @@ -0,0 +1,76 @@ +"""`graphify god-nodes` CLI subcommand (#2004 part 2). + +god_nodes has long been an analyzer + MCP tool + README-advertised capability +but was never wired as a CLI subcommand, so `graphify god_nodes` errored with +"unknown command". These tests pin the subcommand (both spellings), its flags, +and that file nodes are excluded from the ranking. +""" +from __future__ import annotations + +import json + +import networkx as nx +import pytest +from networkx.readwrite import json_graph + +import graphify.__main__ as mainmod + + +def _write_graph(tmp_path): + g = nx.DiGraph() + # A high-degree real entity (not a file/concept node): label != basename. + g.add_node("hub", label="Auth", file_type="code", source_file="auth.py", source_location="L1") + g.add_node("f", label="auth.py", file_type="code", source_file="auth.py", source_location=None) + for i in range(4): + g.add_node(f"caller{i}", label=f"c{i}()", file_type="code", source_file=f"m{i}.py", source_location="L1") + g.add_edge(f"caller{i}", "hub", relation="calls", confidence="EXTRACTED") + g.add_edge("f", "hub", relation="contains", confidence="EXTRACTED") + gp = tmp_path / "graph.json" + gp.write_text(json.dumps(json_graph.node_link_data(g, edges="links")), encoding="utf-8") + return gp + + +def _run(monkeypatch, argv): + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr(mainmod.sys, "argv", argv) + mainmod.main() + + +def test_god_nodes_cli_text_output(monkeypatch, tmp_path, capsys): + gp = _write_graph(tmp_path) + _run(monkeypatch, ["graphify", "god-nodes", "--graph", str(gp)]) + out = capsys.readouterr().out + assert "God nodes (most connected):" in out + assert "Auth" in out + assert "edges" in out + assert "auth.py" not in out # file node excluded from the ranking + + +def test_god_nodes_cli_underscore_alias(monkeypatch, tmp_path, capsys): + # The exact spelling from the issue title. + gp = _write_graph(tmp_path) + _run(monkeypatch, ["graphify", "god_nodes", "--graph", str(gp)]) + assert "Auth" in capsys.readouterr().out + + +def test_god_nodes_cli_top_limits(monkeypatch, tmp_path, capsys): + gp = _write_graph(tmp_path) + _run(monkeypatch, ["graphify", "god-nodes", "--graph", str(gp), "--top", "1"]) + body = capsys.readouterr().out + assert body.count(" edges") == 1 + + +def test_god_nodes_cli_json(monkeypatch, tmp_path, capsys): + gp = _write_graph(tmp_path) + _run(monkeypatch, ["graphify", "god-nodes", "--graph", str(gp), "--json"]) + data = json.loads(capsys.readouterr().out) + assert isinstance(data, list) and data + assert {"id", "label", "degree"} <= set(data[0]) + assert data[0]["label"] == "Auth" + + +def test_god_nodes_cli_missing_graph_errors(monkeypatch, tmp_path, capsys): + with pytest.raises(SystemExit) as exc: + _run(monkeypatch, ["graphify", "god-nodes", "--graph", str(tmp_path / "nope.json")]) + assert exc.value.code == 1 + assert "graph file not found" in capsys.readouterr().err diff --git a/tests/test_home_sandbox.py b/tests/test_home_sandbox.py new file mode 100644 index 000000000..2ddb3ffbc --- /dev/null +++ b/tests/test_home_sandbox.py @@ -0,0 +1,64 @@ +"""Regression tests for the repo-wide HOME sandbox (issue #2168). + +The autouse ``_sandbox_home`` fixture in conftest.py must point every +home-resolution mechanism at a throwaway directory, so installers and +uninstallers exercised by the suite can never delete or rewrite the +developer's real ~/.claude, ~/.gemini, ~/.codebuddy, ~/.copilot, etc. +""" +from __future__ import annotations + +import os +from pathlib import Path + +from graphify.__main__ import claude_uninstall + +# Module import happens during collection, before any fixture runs, so this +# captures the developer's actual home directory for comparison below. +_REAL_HOME = Path(os.path.realpath(os.path.expanduser("~"))) + + +def test_path_home_is_sandboxed(tmp_path_factory): + home = Path(os.path.realpath(Path.home())) + assert home != _REAL_HOME + assert not home.is_relative_to(_REAL_HOME / ".claude") + assert home.name.startswith("sandbox-home") + basetemp = Path(os.path.realpath(tmp_path_factory.getbasetemp())) + assert home.is_relative_to(basetemp) + + +def test_expanduser_is_sandboxed(tmp_path_factory): + expanded = Path(os.path.realpath(os.path.expanduser("~"))) + assert expanded != _REAL_HOME + basetemp = Path(os.path.realpath(tmp_path_factory.getbasetemp())) + assert expanded.is_relative_to(basetemp) + + +def test_claude_config_dir_escape_hatch_is_cleared(): + assert "CLAUDE_CONFIG_DIR" not in os.environ + assert "XDG_CONFIG_HOME" not in os.environ + + +def test_global_uninstall_is_captured_by_sandbox(tmp_path, tmp_path_factory): + """Global skill deletes land inside the sandbox home, never the real one. + + Since #2215, `claude_uninstall(project_dir)` is project-scoped and must NOT + touch the global tree; the global delete now requires either a bare call or + an explicit `remove_user_skill=True`. Both scopes are exercised here so the + sandbox (#2168) is still proven to capture the global delete. + """ + skill = Path.home() / ".claude" / "skills" / "graphify" / "SKILL.md" + skill.parent.mkdir(parents=True) + skill.write_text("# graphify skill (sandbox copy)\n", encoding="utf-8") + + project_dir = tmp_path / "some-project" + project_dir.mkdir() + + # Project-scoped call: the global tree in the sandbox home must survive. + claude_uninstall(project_dir) + assert skill.exists(), "project-scoped uninstall deleted the global skill (#2215 trap)" + + # Explicit global opt-in: the delete happens, and lands in the sandbox. + claude_uninstall(project_dir, remove_user_skill=True) + assert not skill.exists(), "global skill delete was not captured by the sandbox" + # And the sandbox home itself is still inside pytest's tmp area. + assert Path.home().is_relative_to(tmp_path_factory.getbasetemp()) diff --git a/tests/test_hook_guard.py b/tests/test_hook_guard.py new file mode 100644 index 000000000..5146ef250 --- /dev/null +++ b/tests/test_hook_guard.py @@ -0,0 +1,280 @@ +"""Rigorous edge-case coverage for the `graphify hook-guard` subcommand (#522). + +Covers the shell-agnostic PreToolUse/BeforeTool guard that replaced the inline +bash hooks: the search/read detection matrix, the gemini BeforeTool contract, +fail-open behavior, output-dir overrides, subcommand dispatch, exit codes, and +UTF-8 (em dash) byte fidelity. Detection is exercised by calling _run_hook_guard +directly (hermetic, fast); dispatch/exit/encoding go through a real subprocess. +""" +import io +import json +import os +import subprocess +import sys + +import pytest + +from graphify import __main__ as m + + +# --------------------------------------------------------------------------- # +# Direct-call harness: hermetic w.r.t. the ambient GRAPHIFY_OUT env. +# --------------------------------------------------------------------------- # +def _invoke(kind, payload, tmp_path, monkeypatch, *, graph=True, out_name="graphify-out"): + monkeypatch.setattr("graphify.paths.GRAPHIFY_OUT", out_name) + monkeypatch.setattr("graphify.paths.GRAPHIFY_OUT_NAME", out_name) + monkeypatch.chdir(tmp_path) + if graph: + (tmp_path / out_name).mkdir(parents=True, exist_ok=True) + (tmp_path / out_name / "graph.json").write_text("{}", encoding="utf-8") + + if isinstance(payload, (bytes, bytearray)): + data = bytes(payload) + elif payload is None: + data = b"" + else: + data = json.dumps(payload).encode("utf-8") + + class _Stdin: + def __init__(self, b): + self.buffer = io.BytesIO(b) + + monkeypatch.setattr(sys, "stdin", _Stdin(data)) + buf = io.StringIO() + monkeypatch.setattr(sys, "stdout", buf) + m._run_hook_guard(kind) + return buf.getvalue() + + +# --------------------------------------------------------------------------- # +# search: commands that MUST nudge (mirror the old *grep*/token globs) +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("command", [ + "grep -rn foo .", + "pgrep -f server", # contains 'grep' + "egrep pattern file", # contains 'grep' + "fgrep lit file", # contains 'grep' + "ls -la | grep foo", # piped + "ripgrep thing", + "rg pattern src/", + "find . -name '*.py'", + "fd bar", + "ack needle", + "ag needle", +]) +def test_search_nudges(command, tmp_path, monkeypatch): + out = _invoke("search", {"tool_input": {"command": command}}, tmp_path, monkeypatch) + assert "graphify query" in out, f"{command!r} should nudge" + assert json.loads(out)["hookSpecificOutput"]["hookEventName"] == "PreToolUse" + + +# --------------------------------------------------------------------------- # +# search: commands / inputs that MUST stay silent +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("command", [ + "", # empty + "ls -la", + "git status", + "cat README.md", + "python app.py", + "cd findings && ls", # 'find' without a trailing space is not a match + "manage db migrate", # 'ag' mid-word, no 'ag ' token + "echo hello", +]) +def test_search_silent(command, tmp_path, monkeypatch): + out = _invoke("search", {"tool_input": {"command": command}}, tmp_path, monkeypatch) + assert out.strip() == "", f"{command!r} should be silent" + + +def test_search_silent_without_graph(tmp_path, monkeypatch): + out = _invoke("search", {"tool_input": {"command": "grep x"}}, tmp_path, monkeypatch, graph=False) + assert out.strip() == "" + + +def test_search_missing_command_key(tmp_path, monkeypatch): + out = _invoke("search", {"tool_input": {}}, tmp_path, monkeypatch) + assert out.strip() == "" + + +def test_search_non_string_command_is_silent(tmp_path, monkeypatch): + out = _invoke("search", {"tool_input": {"command": 123}}, tmp_path, monkeypatch) + assert out.strip() == "" + + +def test_search_top_level_command_without_tool_input(tmp_path, monkeypatch): + # Some hosts pass the tool payload flat (no "tool_input" wrapper). + out = _invoke("search", {"command": "grep x"}, tmp_path, monkeypatch) + assert "graphify query" in out + + +def test_search_non_dict_tool_input_is_silent(tmp_path, monkeypatch): + out = _invoke("search", {"tool_input": "grep foo"}, tmp_path, monkeypatch) + assert out.strip() == "" + + +# --------------------------------------------------------------------------- # +# read: file targets that MUST nudge +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("tool_input", [ + {"file_path": "src/app.py"}, + {"file_path": "pkg/mod.ts"}, + {"file_path": "src/App.vue"}, + {"file_path": "src/Hero.astro"}, + {"file_path": "src/Card.svelte"}, + {"file_path": "SRC/APP.PY"}, # uppercase extension + {"file_path": "src/a.test.tsx"}, # multi-dot -> .tsx + {"file_path": "lib/foo.min.js"}, # multi-dot -> .js + {"file_path": r"src\components\app.py"}, # windows backslashes + {"pattern": "**/*.py", "path": "src"}, # glob pattern + {"pattern": "**/*.astro"}, +]) +def test_read_nudges(tool_input, tmp_path, monkeypatch): + out = _invoke("read", {"tool_input": tool_input}, tmp_path, monkeypatch) + assert "graphify query" in out, f"{tool_input!r} should nudge" + + +# --------------------------------------------------------------------------- # +# read: targets that MUST stay silent +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("tool_input", [ + {"file_path": "package.json"}, # .json must not match .js + {"file_path": "tsconfig.json"}, + {"file_path": "data.geojson"}, + {"file_path": "uv.lock"}, + {"file_path": "logo.png"}, + {"file_path": "data.bin"}, + {"file_path": ".gitignore"}, + {"file_path": "Makefile"}, # no extension + {"file_path": "my.ts/file"}, # extension on a directory segment + {"file_path": "graphify-out/GRAPH_REPORT.md"}, # the graph's own output + {"file_path": ""}, + {}, # nothing at all +]) +def test_read_silent(tool_input, tmp_path, monkeypatch): + out = _invoke("read", {"tool_input": tool_input}, tmp_path, monkeypatch) + assert out.strip() == "", f"{tool_input!r} should be silent" + + +def test_read_silent_without_graph(tmp_path, monkeypatch): + out = _invoke("read", {"tool_input": {"file_path": "src/app.py"}}, tmp_path, monkeypatch, graph=False) + assert out.strip() == "" + + +def test_read_non_dict_tool_input_is_silent(tmp_path, monkeypatch): + out = _invoke("read", {"tool_input": ["src/app.py"]}, tmp_path, monkeypatch) + assert out.strip() == "" + + +def test_read_respects_custom_output_dir_name(tmp_path, monkeypatch): + # A source file living under a CUSTOM output dir name must be suppressed too, + # not just the literal 'graphify-out/'. + out = _invoke("read", {"tool_input": {"file_path": "build-out/report.py"}}, + tmp_path, monkeypatch, graph=True, out_name="build-out") + assert out.strip() == "" + + +def test_read_nudges_source_outside_custom_output_dir(tmp_path, monkeypatch): + out = _invoke("read", {"tool_input": {"file_path": "src/app.py"}}, + tmp_path, monkeypatch, graph=True, out_name="build-out") + assert "graphify query" in out + + +# --------------------------------------------------------------------------- # +# fail-open: malformed / empty stdin never crashes or blocks +# --------------------------------------------------------------------------- # +@pytest.mark.parametrize("kind", ["search", "read"]) +@pytest.mark.parametrize("raw", [b"not json at all", b"", b"[1,2,3]", b"\xff\xfe\x00bad"]) +def test_fail_open_on_bad_stdin(kind, raw, tmp_path, monkeypatch): + out = _invoke(kind, raw, tmp_path, monkeypatch) + assert out.strip() == "" + + +def test_search_out_path_error_is_swallowed(tmp_path, monkeypatch): + # If the graph-existence check itself throws, the guard stays silent (never + # blocks the tool). + def _boom(*a, **k): + raise OSError("boom") + monkeypatch.setattr("graphify.paths.out_path", _boom) + out = _invoke("search", {"tool_input": {"command": "grep x"}}, tmp_path, monkeypatch) + assert out.strip() == "" + + +# --------------------------------------------------------------------------- # +# gemini: BeforeTool contract (always allow; nudge only when a graph exists) +# --------------------------------------------------------------------------- # +def test_gemini_allow_with_nudge(tmp_path, monkeypatch): + out = _invoke("gemini", None, tmp_path, monkeypatch, graph=True) + payload = json.loads(out) + assert payload["decision"] == "allow" + assert "graphify query" in payload["additionalContext"] + + +def test_gemini_allow_without_graph(tmp_path, monkeypatch): + out = _invoke("gemini", None, tmp_path, monkeypatch, graph=False) + payload = json.loads(out) + assert payload == {"decision": "allow"} + + +def test_gemini_always_allows_even_when_check_throws(tmp_path, monkeypatch): + def _boom(*a, **k): + raise OSError("boom") + monkeypatch.setattr("graphify.paths.out_path", _boom) + out = _invoke("gemini", None, tmp_path, monkeypatch, graph=True) + assert json.loads(out) == {"decision": "allow"} + + +# --------------------------------------------------------------------------- # +# subcommand dispatch, exit codes, and UTF-8 fidelity (real subprocess) +# --------------------------------------------------------------------------- # +def _env(): + e = dict(os.environ) + e.pop("GRAPHIFY_OUT", None) + return e + + +def _cli(args, tmp_path, stdin=""): + return subprocess.run( + [sys.executable, "-m", "graphify", *args], + input=stdin, capture_output=True, text=True, cwd=tmp_path, env=_env(), + ) + + +def test_dispatch_missing_mode_exits_zero_silent(tmp_path): + r = _cli(["hook-guard"], tmp_path, stdin="{}") + assert r.returncode == 0 + assert r.stdout.strip() == "" + + +def test_dispatch_unknown_mode_exits_zero_silent(tmp_path): + r = _cli(["hook-guard", "bogus"], tmp_path, stdin="{}") + assert r.returncode == 0 + assert r.stdout.strip() == "" + + +@pytest.mark.parametrize("args,stdin", [ + (["hook-guard", "search"], '{"tool_input":{"command":"grep x"}}'), + (["hook-guard", "read"], '{"tool_input":{"file_path":"a.py"}}'), + (["hook-guard", "gemini"], ""), +]) +def test_dispatch_always_exits_zero(args, stdin, tmp_path): + # even with a graph present (nudge path), exit code must be 0 (never blocks) + (tmp_path / "graphify-out").mkdir() + (tmp_path / "graphify-out" / "graph.json").write_text("{}", encoding="utf-8") + r = _cli(args, tmp_path, stdin=stdin) + assert r.returncode == 0 + + +def test_read_nudge_em_dash_survives_utf8(tmp_path): + # The read nudge contains an em dash; the emitted bytes must be valid UTF-8 + # and parse back cleanly (guards the ensure_ascii=False + stdout reconfigure). + (tmp_path / "graphify-out").mkdir() + (tmp_path / "graphify-out" / "graph.json").write_text("{}", encoding="utf-8") + r = subprocess.run( + [sys.executable, "-m", "graphify", "hook-guard", "read"], + input=b'{"tool_input":{"file_path":"src/app.py"}}', + capture_output=True, cwd=tmp_path, env=_env(), + ) + assert r.returncode == 0 + text = r.stdout.decode("utf-8") # raises if not valid UTF-8 + payload = json.loads(text) + assert "—" in payload["hookSpecificOutput"]["additionalContext"] # em dash preserved diff --git a/tests/test_hook_strict.py b/tests/test_hook_strict.py new file mode 100644 index 000000000..1b34d13f9 --- /dev/null +++ b/tests/test_hook_strict.py @@ -0,0 +1,202 @@ +"""Strict-mode hook-guard: opt-in block-then-nudge + #1840 gating. + +The strict guard (Claude Code Read only) denies the FIRST raw read of an indexed, +in-project, fresh source file per session, then downgrades to the soft nudge — so +it can never strand an agent. #1840: out-of-project reads are ignored and a graph +that is stale for the target softens to a non-mandatory nudge. Everything defaults +to the historical soft nudge unless strict is explicitly enabled. +""" +import io +import json +import os +import sys +import time + +import pytest + +import graphify.cli as cli + + +def _fixture(tmp_path, *, indexed=True, fresh=True): + """A project with graphify-out/graph.json + manifest and one source file. + ``fresh`` makes the graph newer than the source (not stale).""" + src = tmp_path / "src" + src.mkdir() + f = src / "mod.py" + f.write_text("def x():\n return 1\n", encoding="utf-8") + out = tmp_path / "graphify-out" + out.mkdir() + (out / "manifest.json").write_text( + json.dumps({"src/mod.py": {"mtime": 1}} if indexed else {"other/z.py": {"mtime": 1}}), + encoding="utf-8", + ) + time.sleep(0.02) + (out / "graph.json").write_text('{"nodes":[],"links":[]}', encoding="utf-8") + if not fresh: + time.sleep(0.02) + f.write_text("def x():\n return 2\n", encoding="utf-8") # source now newer -> stale + return f + + +def _invoke(kind, payload, tmp_path, monkeypatch, *, strict=False, env=None): + monkeypatch.chdir(tmp_path) + for k, v in (env or {}).items(): + monkeypatch.setenv(k, v) + data = json.dumps(payload).encode() if not isinstance(payload, (bytes, bytearray)) else bytes(payload) + + class _Stdin: + buffer = io.BytesIO(data) + monkeypatch.setattr(sys, "stdin", _Stdin()) + buf = io.StringIO() + monkeypatch.setattr(sys, "stdout", buf) + cli._run_hook_guard(kind, strict=strict) + return buf.getvalue() + + +def _read(fpath, sid="s1"): + return {"session_id": sid, "tool_name": "Read", "tool_input": {"file_path": str(fpath)}} + + +def _is_deny(out): + return out.strip() != "" and json.loads(out).get("hookSpecificOutput", {}).get("permissionDecision") == "deny" + + +def test_strict_first_read_denies_then_nudges(tmp_path, monkeypatch): + f = _fixture(tmp_path) + out1 = _invoke("read", _read(f), tmp_path, monkeypatch, strict=True) + assert _is_deny(out1) + assert "graphify query" in json.loads(out1)["hookSpecificOutput"]["permissionDecisionReason"] + # marker created + assert (tmp_path / "graphify-out" / "cache" / "hook_sessions" / "s1.denied").exists() + # same session again -> soft nudge, not a second deny + out2 = _invoke("read", _read(f), tmp_path, monkeypatch, strict=True) + assert not _is_deny(out2) and "MANDATORY" in out2 + + +def test_strict_new_session_denies_again(tmp_path, monkeypatch): + f = _fixture(tmp_path) + _invoke("read", _read(f, "sA"), tmp_path, monkeypatch, strict=True) + out = _invoke("read", _read(f, "sB"), tmp_path, monkeypatch, strict=True) + assert _is_deny(out) + + +def test_fresh_query_stamp_suppresses_deny(tmp_path, monkeypatch): + f = _fixture(tmp_path) + stamp = tmp_path / "graphify-out" / "cache" / "last_query_stamp" + stamp.parent.mkdir(parents=True, exist_ok=True) + stamp.write_text(str(time.time()), encoding="utf-8") + out = _invoke("read", _read(f), tmp_path, monkeypatch, strict=True) + assert not _is_deny(out) and "MANDATORY" in out + + +def test_expired_query_stamp_still_denies(tmp_path, monkeypatch): + f = _fixture(tmp_path) + stamp = tmp_path / "graphify-out" / "cache" / "last_query_stamp" + stamp.parent.mkdir(parents=True, exist_ok=True) + stamp.write_text("old", encoding="utf-8") + old = time.time() - 10_000 + os.utime(stamp, (old, old)) + out = _invoke("read", _read(f), tmp_path, monkeypatch, strict=True, env={"GRAPHIFY_HOOK_STRICT_TTL": "1800"}) + assert _is_deny(out) + + +def test_soft_mode_never_denies(tmp_path, monkeypatch): + f = _fixture(tmp_path) + out = _invoke("read", _read(f), tmp_path, monkeypatch, strict=False) + assert not _is_deny(out) and "MANDATORY" in out + + +def test_env_forces_strict_on(tmp_path, monkeypatch): + f = _fixture(tmp_path) + out = _invoke("read", _read(f), tmp_path, monkeypatch, strict=False, env={"GRAPHIFY_HOOK_STRICT": "1"}) + assert _is_deny(out) + + +def test_env_kills_strict(tmp_path, monkeypatch): + f = _fixture(tmp_path) + out = _invoke("read", _read(f), tmp_path, monkeypatch, strict=True, env={"GRAPHIFY_HOOK_STRICT": "0"}) + assert not _is_deny(out) + + +def test_out_of_project_read_silenced(tmp_path, monkeypatch): + _fixture(tmp_path) + payload = {"session_id": "s1", "tool_name": "Read", "tool_input": {"file_path": "/somewhere/else/x.py"}} + assert _invoke("read", payload, tmp_path, monkeypatch, strict=True).strip() == "" + # soft mode too + assert _invoke("read", payload, tmp_path, monkeypatch, strict=False).strip() == "" + + +def test_stale_graph_softens_never_denies(tmp_path, monkeypatch): + f = _fixture(tmp_path, fresh=False) # source newer than graph + out = _invoke("read", _read(f), tmp_path, monkeypatch, strict=True) + assert not _is_deny(out) + assert "stale" in out.lower() and "MANDATORY" not in out + + +def test_needs_update_flag_softens(tmp_path, monkeypatch): + f = _fixture(tmp_path) + (tmp_path / "graphify-out" / "needs_update").write_text("1", encoding="utf-8") + out = _invoke("read", _read(f), tmp_path, monkeypatch, strict=True) + assert not _is_deny(out) and "stale" in out.lower() + + +def test_glob_never_denies(tmp_path, monkeypatch): + _fixture(tmp_path) + payload = {"session_id": "s1", "tool_name": "Glob", + "tool_input": {"pattern": "**/*.py", "path": str(tmp_path)}} + assert not _is_deny(_invoke("read", payload, tmp_path, monkeypatch, strict=True)) + + +def test_search_never_denies(tmp_path, monkeypatch): + _fixture(tmp_path) + out = _invoke("search", {"session_id": "s1", "tool_input": {"command": "grep -rn foo ."}}, + tmp_path, monkeypatch, strict=True) + assert not _is_deny(out) # search stays a nudge even in strict mode + + +def test_no_session_id_never_denies(tmp_path, monkeypatch): + f = _fixture(tmp_path) + payload = {"tool_name": "Read", "tool_input": {"file_path": str(f)}} # no session_id + assert not _is_deny(_invoke("read", payload, tmp_path, monkeypatch, strict=True)) + + +def test_not_indexed_file_not_denied(tmp_path, monkeypatch): + f = _fixture(tmp_path, indexed=False) # manifest doesn't list src/mod.py + out = _invoke("read", _read(f), tmp_path, monkeypatch, strict=True) + assert not _is_deny(out) + + +def test_fail_open_on_malformed_stdin(tmp_path, monkeypatch): + _fixture(tmp_path) + assert _invoke("read", b"{not json", tmp_path, monkeypatch, strict=True) == "" + + +def test_strict_enabled_env_precedence(): + import os as _os + saved = _os.environ.get("GRAPHIFY_HOOK_STRICT") + try: + _os.environ["GRAPHIFY_HOOK_STRICT"] = "1" + assert cli._hook_strict_enabled(False) is True + _os.environ["GRAPHIFY_HOOK_STRICT"] = "0" + assert cli._hook_strict_enabled(True) is False + _os.environ.pop("GRAPHIFY_HOOK_STRICT", None) + assert cli._hook_strict_enabled(True) is True + assert cli._hook_strict_enabled(False) is False + finally: + if saved is None: + _os.environ.pop("GRAPHIFY_HOOK_STRICT", None) + else: + _os.environ["GRAPHIFY_HOOK_STRICT"] = saved + + +def test_install_hook_carries_strict_flag(): + from graphify.install import _claude_pretooluse_hooks + soft = _claude_pretooluse_hooks(strict=False) + strict = _claude_pretooluse_hooks(strict=True) + read_soft = next(h for h in soft if h["matcher"] == "Read|Glob")["hooks"][0]["command"] + read_strict = next(h for h in strict if h["matcher"] == "Read|Glob")["hooks"][0]["command"] + assert read_soft.endswith("hook-guard read") + assert read_strict.endswith("hook-guard read --strict") + # search hook is unchanged either way + for hooks in (soft, strict): + assert next(h for h in hooks if h["matcher"] == "Bash|Grep")["hooks"][0]["command"].endswith("hook-guard search") diff --git a/tests/test_hooks.py b/tests/test_hooks.py index 6a37bd932..29acc9ead 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -1,5 +1,6 @@ """Tests for hooks.py - git hook install/uninstall.""" import os +import shutil import subprocess from types import SimpleNamespace from pathlib import Path @@ -329,6 +330,31 @@ def test_rebuild_bodies_with_graphify_root_are_valid_python(): ast.parse(body) +@pytest.mark.parametrize( + "name,body", + [("post-commit", _REBUILD_BODY_COMMIT), ("post-checkout", _REBUILD_BODY_CHECKOUT)], +) +def test_rebuild_bodies_arm_a_timeout_without_sigalrm(name, body): + """Windows has no signal.SIGALRM, so the #791 rebuild timeout never armed + there at all (#2148). The fallback has to sit in the else-branch of the + SIGALRM check rather than merely appear somewhere in the body, so that a + watchdog firing unconditionally or on every platform still fails here.""" + fallbacks = [ + node.orelse + for node in ast.walk(ast.parse(body)) + if isinstance(node, ast.If) and "'SIGALRM'" in ast.dump(node.test) and node.orelse + ] + assert fallbacks, f"{name} has no else-branch for the missing-SIGALRM case (#2148)" + dumped = "".join(ast.dump(stmt) for stmt in fallbacks[0]) + assert "attr='Timer'" in dumped, f"{name} fallback does not arm a threading.Timer (#2148)" + assert "attr='_exit'" in dumped, f"{name} fallback does not kill the stuck rebuild (#2148)" + # The fallback logs the timeout itself, because os._exit skips the except + # handler that reports it on the SIGALRM path. Its prefix has to match the + # rest of the body, or the same event reads differently per platform. + prefixes = set(re.findall(r"print\(f'\[([a-z ]+)\]", body)) + assert len(prefixes) == 1, f"{name} mixes log prefixes {sorted(prefixes)} (#2148)" + + def test_detached_launch_targets_graphify_python(): """The launcher must run via the resolved $GRAPHIFY_PYTHON, not a bare `python`, so it uses the same interpreter the detection block selected.""" @@ -424,9 +450,308 @@ def test_probe_prefers_sibling_python_exe_on_windows_layouts(): assert "/python.exe" in _PYTHON_DETECT +def _extract_case_pattern(marker: str) -> str: + """Pull the `*[!...]*` glob portion of a real case arm out of _PYTHON_DETECT + by a unique anchor, so tests run against the emitted text, not a copy.""" + from graphify.hooks import _PYTHON_DETECT + for line in _PYTHON_DETECT.splitlines(): + if marker in line: + return line.strip().split(")")[0] + raise AssertionError(f"case arm containing {marker!r} not found in _PYTHON_DETECT") + + +def _shell_verdict(pattern: str, candidate: str) -> str: + result = subprocess.run( + ["bash", "-c", f'case "$1" in\n{pattern}) echo REJECTED ;;\n*) echo ACCEPTED ;;\nesac', "_", candidate], + capture_output=True, text=True, + ) + # Fail loudly on a malformed case snippet instead of returning "" and + # producing a confusing ACCEPTED/REJECTED mismatch downstream. + assert result.returncode == 0, ( + f"bash exited {result.returncode} for pattern {pattern!r}: {result.stderr.strip()}" + ) + return result.stdout.strip() + + +@pytest.mark.skipif(shutil.which("bash") is None, reason="bash required to exercise emitted glob") +@pytest.mark.parametrize("winpath", [ + r"C:\Users\u\.venv\Scripts\python.exe", + r"C:\Python311\python.exe", +]) +def test_file_path_allowlist_accepts_windows_backslash_path(winpath): + """#2126: the .graphify_python FILE allowlist must accept real Windows paths + at actual shell runtime. Old pattern rejected them due to bash bracket-escape.""" + pattern = _extract_case_pattern('_FROM_FILE=""') + assert _shell_verdict(pattern, winpath) == "ACCEPTED", ( + f"Windows path {winpath!r} rejected by file-path allowlist at shell runtime" + ) + + +@pytest.mark.skipif(shutil.which("bash") is None, reason="bash required to exercise emitted glob") +@pytest.mark.parametrize("shebang_path", [ + r"C:\Users\u\.venv\Scripts\python.exe", +]) +def test_shebang_allowlist_accepts_windows_backslash_path(shebang_path): + """#2126: the shebang-parsed launcher allowlist had no `:` or `\\` at all, so + any Windows-style shebang path was unconditionally emptied. Must ACCEPT now.""" + pattern = _extract_case_pattern('GRAPHIFY_PYTHON="" ;;') + assert _shell_verdict(pattern, shebang_path) == "ACCEPTED", ( + f"Windows shebang path {shebang_path!r} rejected by launcher allowlist" + ) + + +@pytest.mark.skipif(shutil.which("bash") is None, reason="bash required to exercise emitted glob") +@pytest.mark.parametrize("dangerous", ["foo;rm -rf /", "foo`id`", "foo$(id)", "foo$IFS"]) +def test_python_detect_allowlists_still_reject_shell_metacharacters(dangerous): + """Guard against a naive fix (backslash right before `]`) that forms a + `:`-to-`\\` range admitting `;`, backtick, `$`. Both allowlists must reject.""" + for marker in ('_FROM_FILE=""', 'GRAPHIFY_PYTHON="" ;;'): + pattern = _extract_case_pattern(marker) + assert _shell_verdict(pattern, dangerous) == "REJECTED", ( + f"{marker} allowlist wrongly accepted dangerous input {dangerous!r}" + ) + + @pytest.mark.parametrize("name,script", _HOOK_SCRIPTS) def test_hooks_reuse_git_dir_from_env(name, script): """git exports GIT_DIR to hooks, so the rev-parse fallback should only run when the script is invoked by hand — each extra git exec costs 1s+ on AV-scanned Windows machines and lands in the commit's foreground.""" assert "GIT_DIR=${GIT_DIR:-" in script, f"{name} always re-runs git rev-parse" + + +@pytest.mark.parametrize("name,script", _HOOK_SCRIPTS) +def test_hooks_honor_skip_env(name, script): + """GRAPHIFY_SKIP_HOOK=1 must suppress BOTH hooks. post-checkout previously + lacked the check, so the var stopped commit rebuilds but not branch-switch + ones (#1809).""" + assert '[ "${GRAPHIFY_SKIP_HOOK:-0}" = "1" ] && exit 0' in script, ( + f"{name} does not honor GRAPHIFY_SKIP_HOOK" + ) + + +@pytest.mark.parametrize("name,script", _HOOK_SCRIPTS) +def test_hooks_skip_linked_worktrees(name, script): + """Both hooks must short-circuit in a linked worktree (git-dir != common-dir), + and must compare ABSOLUTE paths so the primary checkout (where --git-common-dir + is the relative ".git") is not false-positived and wrongly skipped (#1809, #1806).""" + assert script.count("_GFY_GITDIR=") == 1, f"{name} guard not present exactly once" + assert "git rev-parse --git-common-dir" in script + # absolute-normalized compare, not a raw string compare of git output + assert 'cd "$(git rev-parse --git-dir 2>/dev/null)" 2>/dev/null && pwd' in script + assert '[ "$_GFY_GITDIR" != "$_GFY_COMMONDIR" ]' in script + + +def _worktree_guard_snippet() -> str: + from graphify.hooks import _WORKTREE_GUARD + return _WORKTREE_GUARD + "echo RAN\n" + + +def test_worktree_guard_runs_on_primary_skips_linked(tmp_path): + """End-to-end against a real `git worktree`: the guard falls through on the + primary checkout and exits early inside a linked worktree (#1809, #1806).""" + if shutil.which("git") is None: # pragma: no cover + pytest.skip("git not available") + primary = tmp_path / "primary" + primary.mkdir() + + def _git(*args, cwd): + subprocess.run(["git", *args], cwd=cwd, check=True, + capture_output=True, text=True) + + _git("init", "-q", ".", cwd=primary) + _git("config", "user.email", "t@t.co", cwd=primary) + _git("config", "user.name", "t", cwd=primary) + (primary / "a.txt").write_text("x") + _git("add", "-A", cwd=primary) + _git("commit", "-qm", "init", cwd=primary) + linked = tmp_path / "linked" + _git("worktree", "add", "-q", str(linked), "-b", "feature", cwd=primary) + + snippet = _worktree_guard_snippet() + r_primary = subprocess.run(["sh", "-c", snippet], cwd=primary, + capture_output=True, text=True) + r_linked = subprocess.run(["sh", "-c", snippet], cwd=linked, + capture_output=True, text=True) + assert "RAN" in r_primary.stdout, "guard wrongly skipped the primary checkout" + assert "RAN" not in r_linked.stdout, "guard failed to skip the linked worktree" + + +# ── #1907: duplicate keys in .git/config must not trigger spurious warnings ── + +def _append_duplicate_config_entries(repo: Path) -> None: + """Append git-legal duplicate keys/sections (as VS Code writes them).""" + cfg = repo / ".git" / "config" + cfg.write_text( + cfg.read_text(encoding="utf-8") + + '[remote "origin"]\n' + + "\tfetch = +refs/heads/*:refs/remotes/origin/*\n" + + "\tfetch = +refs/heads/*:refs/remotes/origin/*\n" + + "[core]\n" + + "\tignorecase = true\n", + encoding="utf-8", + ) + + +def test_hooks_dir_no_warning_on_duplicate_config_keys(tmp_path, capsys): + """git legally allows duplicate keys and repeated sections in .git/config; + a strict configparser raised DuplicateOptionError/DuplicateSectionError and + printed a spurious 'could not read core.hooksPath' warning on every hook + command (#1907). _hooks_dir must resolve cleanly with no stderr noise.""" + repo = _make_git_repo(tmp_path) + _append_duplicate_config_entries(repo) + d = _hooks_dir(repo) + err = capsys.readouterr().err + assert "could not read core.hooksPath" not in err + assert d == (repo / ".git" / "hooks").resolve() + + +def test_hooks_dir_duplicate_config_keys_honor_custom_hookspath(tmp_path, capsys): + """With duplicate keys present, a custom core.hooksPath must still be + honored (no fall-through to .git/hooks) and no warning printed (#1907).""" + repo = _make_git_repo(tmp_path) + _set_hookspath(repo, ".husky") + _append_duplicate_config_entries(repo) + d = _hooks_dir(repo) + err = capsys.readouterr().err + assert "could not read core.hooksPath" not in err + assert d == (repo / ".husky").resolve() + + +# ── #1902: hook install must register the graph.json union merge driver ───── + +def test_install_registers_merge_driver(tmp_path): + """install() must set merge.graphify.* via git config and add the + .gitattributes line that README/CHANGELOG 0.7.0 document (#1902).""" + repo = _make_git_repo(tmp_path) + result = install(repo) + res = subprocess.run( + ["git", "-C", str(repo), "config", "--get", "merge.graphify.driver"], + capture_output=True, text=True, + ) + assert res.returncode == 0 + driver = res.stdout.strip() + assert driver + assert "merge-driver %O %A %B" in driver + attrs = (repo / ".gitattributes").read_text(encoding="utf-8") + assert any( + "graph.json" in line and "merge=graphify" in line + for line in attrs.splitlines() + ) + assert "merge driver" in result + + +def test_install_merge_driver_idempotent(tmp_path): + """Running install twice must not duplicate the .gitattributes line.""" + repo = _make_git_repo(tmp_path) + install(repo) + install(repo) + lines = (repo / ".gitattributes").read_text(encoding="utf-8").splitlines() + matches = [l for l in lines if "merge=graphify" in l] + assert len(matches) == 1 + + +def test_install_preserves_existing_gitattributes(tmp_path): + """A pre-existing .gitattributes entry must survive install (no clobber).""" + repo = _make_git_repo(tmp_path) + (repo / ".gitattributes").write_text("*.png binary\n", encoding="utf-8") + install(repo) + content = (repo / ".gitattributes").read_text(encoding="utf-8") + assert "*.png binary" in content + assert "merge=graphify" in content + + +def test_uninstall_removes_merge_driver_keeps_other_attrs(tmp_path): + """uninstall() must unset merge.graphify.* and remove only the graphify + .gitattributes line, keeping the file when other entries exist.""" + repo = _make_git_repo(tmp_path) + (repo / ".gitattributes").write_text("*.png binary\n", encoding="utf-8") + install(repo) + uninstall(repo) + res = subprocess.run( + ["git", "-C", str(repo), "config", "--get", "merge.graphify.driver"], + capture_output=True, text=True, + ) + assert res.returncode != 0 + content = (repo / ".gitattributes").read_text(encoding="utf-8") + assert "*.png binary" in content + assert "merge=graphify" not in content + + +@pytest.mark.parametrize("exe", [ + r"C:\Users\First Last\AppData\Roaming\uv\tools\graphifyy\Scripts\python.exe", + r"C:\Program Files\Python312\python.exe", + "/home/first last/.local/share/uv/tools/graphifyy/bin/python", +]) +def test_pinned_python_accepts_paths_containing_spaces(exe, monkeypatch): + """#2166: a space must not empty the pin. + + The install-time allowlist had no space, so `sys.executable` under any Windows + profile whose name contains one (`C:\\Users\\First Last\\...`, or the very common + `C:\\Program Files\\...`) was rejected wholesale and the hook shipped `_PINNED=''`. + Every interpreter probe then failed and each commit no-op'd with the "could not + locate a Python" warning, so the graph never rebuilt. + """ + import sys as _sys + + from graphify.hooks import _pinned_python + + monkeypatch.setattr(_sys, "executable", exe) + assert _pinned_python() == exe, "a path containing a space must still be pinned" + + +@pytest.mark.parametrize("exe", [ + r"C:\Users\evil\python.exe; rm -rf /", + "/tmp/py`id`", + "/tmp/py$(id)", + "/tmp/py$IFS", + r"C:\Users\ev'il\python.exe", + '/tmp/py"quote', +]) +def test_pinned_python_still_rejects_shell_metacharacters(exe, monkeypatch): + """Widening the allowlist for spaces (#2166) must not admit anything that can + start a substitution, end the single-quoted assignment, or chain a command.""" + import sys as _sys + + from graphify.hooks import _pinned_python + + monkeypatch.setattr(_sys, "executable", exe) + assert _pinned_python() == "", f"dangerous interpreter path accepted: {exe!r}" + + +def test_merge_driver_quotes_interpreter_with_spaces(tmp_path, monkeypatch): + """#2166: git runs the merge driver through a shell, so a pinned path with a + space has to be quoted or the driver splits into two words and never runs.""" + import subprocess + import sys as _sys + + from graphify.hooks import install + + exe = r"C:\Users\First Last\AppData\Roaming\uv\tools\graphifyy\Scripts\python.exe" + repo = _make_git_repo(tmp_path) + monkeypatch.setattr(_sys, "executable", exe) + install(repo) + + driver = subprocess.run( + ["git", "-C", str(repo), "config", "--get", "merge.graphify.driver"], + capture_output=True, text=True, check=True, + ).stdout.strip() + assert driver.startswith(f'"{exe}"'), f"interpreter not quoted in merge driver: {driver!r}" + assert driver.endswith("-m graphify merge-driver %O %A %B") + + +def test_install_pins_interpreter_path_with_spaces(tmp_path, monkeypatch): + """#2166 end to end: the emitted hooks must carry the real interpreter, not ''.""" + import sys as _sys + + from graphify.hooks import install + + exe = r"C:\Users\First Last\AppData\Roaming\uv\tools\graphifyy\Scripts\python.exe" + repo = _make_git_repo(tmp_path) + monkeypatch.setattr(_sys, "executable", exe) + install(repo) + + for name in ("post-commit", "post-checkout"): + script = (repo / ".git" / "hooks" / name).read_text() + assert f"_PINNED='{exe}'" in script, f"{name} did not pin the spaced interpreter" + assert "_PINNED=''" not in script, f"{name} pinned an empty interpreter (#2166)" diff --git a/tests/test_import_self_loops.py b/tests/test_import_self_loops.py new file mode 100644 index 000000000..3716baa3b --- /dev/null +++ b/tests/test_import_self_loops.py @@ -0,0 +1,107 @@ +from pathlib import Path + +import pytest + +from graphify.build import build_from_json +from graphify.extract import extract + + +def _write(path: Path, source: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(source, encoding="utf-8") + return path + + +def _import_self_loops(result: dict) -> list[dict]: + return [ + edge + for edge in result["edges"] + if edge.get("relation") in {"imports", "imports_from", "re_exports"} + and edge.get("source") == edge.get("target") + ] + + +def _built_import_self_loops(result: dict) -> list[tuple[str, str, dict]]: + graph = build_from_json(result, directed=True) + return [ + (source, target, data) + for source, target, data in graph.edges(data=True) + if source == target + and data.get("relation") in {"imports", "imports_from", "re_exports"} + ] + + +@pytest.mark.parametrize( + ("relative_path", "source"), + [ + ("src/contracting/stdlib/builtins.py", "import builtins\n"), + ("playground/services/contracting.py", "from contracting import constants\n"), + ], +) +def test_python_external_import_matching_current_basename_has_no_self_loop( + tmp_path: Path, + relative_path: str, + source: str, +) -> None: + module = _write(tmp_path / relative_path, source) + + result = extract([module], cache_root=tmp_path, parallel=False) + + assert _import_self_loops(result) == [] + assert _built_import_self_loops(result) == [] + + +@pytest.mark.parametrize( + ("relative_path", "source"), + [ + ( + "packages/compiler/src/fixture.rs", + "mod tests { use crate::fixture::Fixture; }\n", + ), + ( + "packages/zk/src/poseidon.rs", + "use ark_crypto_primitives::sponge::poseidon::{PoseidonConfig, PoseidonSponge};\n", + ), + ], +) +def test_rust_import_matching_current_basename_has_no_self_loop( + tmp_path: Path, + relative_path: str, + source: str, +) -> None: + module = _write(tmp_path / relative_path, source) + + result = extract([module], cache_root=tmp_path, parallel=False) + + assert _import_self_loops(result) == [] + assert _built_import_self_loops(result) == [] + + +def test_recursive_call_self_loop_is_preserved() -> None: + result = { + "nodes": [ + { + "id": "module_recurse", + "label": "recurse()", + "file_type": "code", + "source_file": "module.py", + "source_location": "L1", + "_origin": "ast", + } + ], + "edges": [ + { + "source": "module_recurse", + "target": "module_recurse", + "relation": "calls", + "confidence": "EXTRACTED", + "source_file": "module.py", + "source_location": "L2", + } + ], + } + + graph = build_from_json(result, directed=True) + + assert graph.has_edge("module_recurse", "module_recurse") + assert graph["module_recurse"]["module_recurse"]["relation"] == "calls" diff --git a/tests/test_incomplete_build_guard.py b/tests/test_incomplete_build_guard.py new file mode 100644 index 000000000..ced50bc3f --- /dev/null +++ b/tests/test_incomplete_build_guard.py @@ -0,0 +1,232 @@ +"""Tests for the incomplete-build shrink-guard on `graphify extract`. + +A full build writes the graph with `to_json(..., force=True)`, which bypasses the +#479 shrink guard. When this run's extraction was incomplete (an AST pass crashed +or some semantic chunks failed), forcing the write can silently overwrite a good +complete graph with a smaller partial one. The build now drops back to the shrink +guard (force=False) on an incomplete run — unless `--allow-partial` is passed — +and exits non-zero (before writing the manifest) if the guard refuses. +""" +from __future__ import annotations + +import pytest + +import graphify.__main__ as mainmod + + +def _make_docs_corpus(tmp_path): + # Docs-only corpus: no code files, so AST extraction is skipped and the only + # driver of incompleteness is the (stubbed) semantic chunk run. + (tmp_path / "README.md").write_text("# Notes\nThe entry point overview.\n") + (tmp_path / "GUIDE.md").write_text("# Guide\nHow to use the thing.\n") + return tmp_path + + +def _seed_to_json_recorder(monkeypatch, *, returns=True): + """Patch export.to_json to record the ``force`` it was called with and return + a fixed bool (True = wrote, False = shrink guard refused).""" + rec = {"called": False, "force": None} + + def _stub(G, communities, output_path, *, force=False, **kwargs): + rec["called"] = True + rec["force"] = force + return returns + + monkeypatch.setattr("graphify.export.to_json", _stub) + return rec + + +def _arm_extract(monkeypatch, tmp_path, *, chunk_total, chunk_succeeded, extra_argv=()): + corpus = _make_docs_corpus(tmp_path) + out_dir = tmp_path / "out" + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") + + def _stub_corpus(paths, **kwargs): + on_chunk = kwargs.get("on_chunk_done") + if on_chunk: + for i in range(chunk_succeeded): + on_chunk(i, chunk_total, {"nodes": [], "edges": [], "hyperedges": []}) + return { + "nodes": [{"id": "s1", "source_file": str(corpus / "README.md"), + "file_type": "document", "label": "Notes"}], + "edges": [], "hyperedges": [], "input_tokens": 10, "output_tokens": 5, + } + + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _stub_corpus) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, "argv", + ["graphify", "extract", str(corpus), "--backend", "claude", + "--out", str(out_dir), *extra_argv], + ) + return out_dir + + +def test_partial_extraction_refuses_to_shrink_existing_graph(monkeypatch, tmp_path, capsys): + # 1 of 3 chunks succeeded -> incomplete; the shrink guard refuses (returns False). + rec = _seed_to_json_recorder(monkeypatch, returns=False) + out_dir = _arm_extract(monkeypatch, tmp_path, chunk_total=3, chunk_succeeded=1) + + with pytest.raises(SystemExit) as exc: + mainmod.main() + + assert exc.value.code == 1 + assert rec["called"] and rec["force"] is False, "incomplete build must not force the write" + err = capsys.readouterr().err + assert "Refusing to overwrite" in err + # The manifest must not be stamped for a graph we declined to write. + assert not (out_dir / "graphify-out" / "manifest.json").exists() + + +def test_partial_extraction_writes_when_not_shrinking(monkeypatch, tmp_path): + # Incomplete run, but the new graph is not smaller -> the guard permits the + # write. force is still False (guard active), and the CLI does not exit 1. + rec = _seed_to_json_recorder(monkeypatch, returns=True) + _arm_extract(monkeypatch, tmp_path, chunk_total=3, chunk_succeeded=1) + + mainmod.main() # no SystemExit + + assert rec["called"] and rec["force"] is False + + +def test_allow_partial_forces_write_despite_incomplete(monkeypatch, tmp_path): + rec = _seed_to_json_recorder(monkeypatch, returns=True) + _arm_extract(monkeypatch, tmp_path, chunk_total=3, chunk_succeeded=1, + extra_argv=["--allow-partial"]) + + mainmod.main() + + assert rec["called"] and rec["force"] is True, "--allow-partial must restore force=True" + + +def test_complete_extraction_keeps_force_write(monkeypatch, tmp_path): + # All chunks succeeded -> a complete build legitimately keeps force=True so a + # genuine dedup/deletion shrink still overwrites. + rec = _seed_to_json_recorder(monkeypatch, returns=True) + _arm_extract(monkeypatch, tmp_path, chunk_total=1, chunk_succeeded=1) + + mainmod.main() + + assert rec["called"] and rec["force"] is True + + +def _seed_existing_graph(gout, n): + import json + gout.mkdir(parents=True, exist_ok=True) + (gout / "graph.json").write_text( + json.dumps({"nodes": [{"id": f"keep{i}", "label": f"k{i}"} for i in range(n)], + "links": []}), + encoding="utf-8", + ) + + +def _arm_no_cluster(monkeypatch, tmp_path, *, extra_argv=()): + corpus = _make_docs_corpus(tmp_path) + out_dir = tmp_path / "out" + gout = out_dir / "graphify-out" + _seed_existing_graph(gout, 5) # existing complete graph + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test-fake-key") + + def _stub_corpus(paths, **kwargs): + on_chunk = kwargs.get("on_chunk_done") + if on_chunk: + on_chunk(0, 3, {"nodes": [], "edges": [], "hyperedges": []}) # 1 of 3 -> partial + return {"nodes": [{"id": "s1", "source_file": str(corpus / "README.md"), + "file_type": "document", "label": "Notes"}], + "edges": [], "hyperedges": [], "input_tokens": 1, "output_tokens": 1} + + monkeypatch.setattr("graphify.llm.extract_corpus_parallel", _stub_corpus) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, "argv", + ["graphify", "extract", str(corpus), "--backend", "claude", "--no-cluster", + "--out", str(out_dir), *extra_argv], + ) + return gout / "graph.json" + + +def test_no_cluster_incomplete_build_refuses_to_shrink(tmp_path, monkeypatch, capsys): + # --force: the non-incremental raw-dump path, where the shrink guard is the + # only thing standing between a partial 1-node extraction and the existing + # complete 5-node graph. (Incremental runs merge the existing graph forward + # first — #2169 — so a partial run no longer shrinks there; see below.) + import json + graph = _arm_no_cluster(monkeypatch, tmp_path, extra_argv=["--force"]) + + with pytest.raises(SystemExit) as exc: + mainmod.main() + + assert exc.value.code == 1 + assert "Refusing to overwrite" in capsys.readouterr().err + # The existing 5-node graph is untouched — the partial 1-node graph was refused. + assert len(json.loads(graph.read_text())["nodes"]) == 5 + + +def test_no_cluster_incremental_incomplete_build_carries_existing_nodes( + tmp_path, monkeypatch +): + """#2169: an INCREMENTAL --no-cluster run merges the existing graph forward, + so even an incomplete extraction does not shrink the graph — the existing + nodes are carried and this run's partial chunk is added, no guard refusal.""" + import json + graph = _arm_no_cluster(monkeypatch, tmp_path) + + with pytest.raises(SystemExit) as exc: + mainmod.main() + + assert exc.value.code == 0 # the raw --no-cluster path exits 0 on success + ids = {n["id"] for n in json.loads(graph.read_text())["nodes"]} + assert {f"keep{i}" for i in range(5)} <= ids, ids + assert "s1" in ids, ids + + +def test_no_cluster_allow_partial_overwrites(tmp_path, monkeypatch): + import json + graph = _arm_no_cluster( + monkeypatch, tmp_path, extra_argv=["--force", "--allow-partial"] + ) + + with pytest.raises(SystemExit) as exc: + mainmod.main() + + assert exc.value.code == 0 # the raw --no-cluster path exits 0 on success + assert len(json.loads(graph.read_text())["nodes"]) == 1 + + +def test_no_cluster_incomplete_build_fails_closed_on_malformed_existing_graph( + tmp_path, monkeypatch, capsys +): + """A present-but-unparseable existing graph.json (corrupt or mid-write) could + be hiding a complete graph, so an incomplete --no-cluster build must refuse + to overwrite it — matching to_json's #479 fail-closed handling, not the + fail-open 'proceed when we can't count' path. --force: the non-incremental + raw-dump path (the incremental path fails even earlier, at the forward + merge — see the test below).""" + graph = _arm_no_cluster(monkeypatch, tmp_path, extra_argv=["--force"]) + graph.write_text("{corrupt json", encoding="utf-8") # non-empty, unparseable + + with pytest.raises(SystemExit) as exc: + mainmod.main() + + assert exc.value.code == 1 + assert "unparseable" in capsys.readouterr().err + # The corrupt file is left untouched rather than clobbered by the partial build. + assert graph.read_text() == "{corrupt json" + + +def test_no_cluster_incremental_malformed_existing_graph_refuses_merge( + tmp_path, monkeypatch, capsys +): + """#2169: an incremental --no-cluster run must hard-fail on an unparseable + existing graph.json (build_merge's message) instead of raw-dumping this + run's chunks over it.""" + graph = _arm_no_cluster(monkeypatch, tmp_path) + graph.write_text("{corrupt json", encoding="utf-8") # non-empty, unparseable + + with pytest.raises(SystemExit) as exc: + mainmod.main() + + assert exc.value.code == 1 + assert "Cannot read" in capsys.readouterr().err + # The corrupt file is left untouched rather than clobbered. + assert graph.read_text() == "{corrupt json" diff --git a/tests/test_incremental.py b/tests/test_incremental.py index c1df58aca..62c120325 100644 --- a/tests/test_incremental.py +++ b/tests/test_incremental.py @@ -103,6 +103,211 @@ def _edges(graph_json: Path) -> list[dict]: return g.get("links", g.get("edges", [])) +def test_extract_no_cluster_incremental_changed_file_preserves_unchanged_files(tmp_path): + """#2169: an incremental --no-cluster extract of ONE changed file must merge + into the existing graph, not overwrite graph.json with just that file's + chunk — and the changed file's cross-file import edges must keep pointing at + the unchanged target file's canonical node ids, not dangling + absolute-path-derived ones.""" + proj = tmp_path / "proj" + (proj / "app" / "add").mkdir(parents=True) + (proj / "src" / "components").mkdir(parents=True) + (proj / "src" / "components" / "ScanScreen.tsx").write_text( + "export function ScanScreen() {\n return null;\n}\n", encoding="utf-8" + ) + scan_tsx = proj / "app" / "add" / "scan.tsx" + scan_tsx.write_text( + "import {ScanScreen} from '../../src/components/ScanScreen';\n" + "export default ScanScreen;\n", + encoding="utf-8", + ) + + first = _run(["extract", str(proj), "--code-only", "--no-cluster"], tmp_path) + assert first.returncode == 0, first.stderr + gj = proj / "graphify-out" / "graph.json" + base = json.loads(gj.read_text(encoding="utf-8")) + base_ids = {n["id"] for n in base["nodes"]} + # Sanity: importer file, target file, and target symbol all present. + assert { + "app_add_scan", + "src_components_scanscreen", + "src_components_scanscreen_scanscreen", + } <= base_ids, base_ids + + # Change ONLY scan.tsx (harmless comment), then re-run the same command. + scan_tsx.write_text( + scan_tsx.read_text(encoding="utf-8") + "\n// touched\n", encoding="utf-8" + ) + second = _run(["extract", str(proj), "--code-only", "--no-cluster"], tmp_path) + assert second.returncode == 0, second.stderr + # Guard against a silent full rescan masking the merge bug. + assert "incremental scan" in second.stdout.lower(), second.stdout + + after = json.loads(gj.read_text(encoding="utf-8")) + after_ids = {n["id"] for n in after["nodes"]} + # The unchanged file's nodes must survive the incremental raw write. + assert after_ids == base_ids, ( + f"incremental --no-cluster dropped/changed nodes: " + f"missing={base_ids - after_ids}, extra={after_ids - base_ids}" + ) + after_edges = after.get("links", after.get("edges", [])) + # The unchanged file's own edge survives. + assert any( + e.get("relation") == "contains" + and e.get("source") == "src_components_scanscreen" + and e.get("target") == "src_components_scanscreen_scanscreen" + for e in after_edges + ), after_edges + # No dangling endpoints on cross-file edges: the changed file's re-extracted + # imports/re-exports must resolve to the unchanged target's canonical ids, + # not absolute-path-derived ghosts (the extract.py half of #2169). + for e in after_edges: + if e.get("relation") in ("imports_from", "re_exports", "contains", "imports"): + assert e.get("source") in after_ids, f"dangling source: {e}" + assert e.get("target") in after_ids, f"dangling target: {e}" + + +def test_extract_no_cluster_incremental_code_only_preserves_doc_nodes(tmp_path): + """#2169: an incremental --code-only --no-cluster run over a mixed corpus + must carry forward doc-sourced nodes it did not re-extract.""" + proj = tmp_path / "proj" + proj.mkdir() + util = proj / "util.py" + util.write_text("def alpha():\n return 1\n", encoding="utf-8") + (proj / "notes.md").write_text("# Notes\nSome prose.\n", encoding="utf-8") + + first = _run(["extract", str(proj), "--code-only", "--no-cluster"], tmp_path) + assert first.returncode == 0, first.stderr + gj = proj / "graphify-out" / "graph.json" + g = json.loads(gj.read_text(encoding="utf-8")) + assert g.get("nodes"), "first run should produce a non-empty code graph" + + # Seed a doc-sourced node, as a prior (LLM-backed) run would have written. + g["nodes"].append({ + "id": "notes", + "label": "notes.md", + "type": "document", + "source_file": "notes.md", + }) + gj.write_text(json.dumps(g), encoding="utf-8") + + # Change only the code file; the doc node must survive the incremental run. + util.write_text( + "def alpha():\n return 1\n\ndef beta():\n return 2\n", + encoding="utf-8", + ) + second = _run(["extract", str(proj), "--code-only", "--no-cluster"], tmp_path) + assert second.returncode == 0, second.stderr + assert "incremental scan" in second.stdout.lower(), second.stdout + + after = json.loads(gj.read_text(encoding="utf-8")) + after_by_id = {n["id"]: n for n in after["nodes"]} + assert "notes" in after_by_id, ( + f"doc node dropped by incremental --code-only --no-cluster: " + f"{sorted(after_by_id)}" + ) + assert after_by_id["notes"].get("source_file") == "notes.md" + # And the changed code file was actually re-extracted. + assert any("beta" in i for i in after_by_id), sorted(after_by_id) + + +def test_incremental_python_relative_import_target_canonicalizes(tmp_path): + """#2213 (defect 1, shared root with #2211): a Python relative import's + imports_from edge must stamp target_file so the #2169 remap canonicalizes + its target on an incremental extraction where the target file is NOT in + the batch — instead of leaving an absolute-path-derived dangling id.""" + from graphify.extract import extract, _make_id + + # realpath: on macOS the pytest tmp dir can sit behind a symlink + # (/tmp -> /private/tmp); anchor everything on the resolved form so the + # canonical-id assertions are deterministic. + tmp = Path(os.path.realpath(tmp_path)) + pkg = tmp / "pkg" + pkg.mkdir() + (pkg / "b.py").write_text( + "class Thing:\n def go(self):\n return 1\n", encoding="utf-8" + ) + a = pkg / "a.py" + a.write_text( + "from .b import Thing\n\n\ndef use():\n return Thing().go()\n", + encoding="utf-8", + ) + + full = extract([a, pkg / "b.py"], cache_root=tmp) + full_imports = [ + e for e in full["edges"] + if e.get("relation") == "imports_from" + and str(e.get("source_file", "")).endswith("a.py") + ] + assert full_imports, full["edges"] + canonical = full_imports[0]["target"] + assert canonical == "pkg_b", canonical + + # Incremental: only the importer is in the batch (b.py unchanged, so the + # #2169 merge path re-extracts a.py alone). Same cache/scan root. + inc = extract([a], cache_root=tmp) + inc_imports = [ + e for e in inc["edges"] + if e.get("relation") == "imports_from" + and str(e.get("source_file", "")).endswith("a.py") + ] + assert inc_imports, inc["edges"] + assert inc_imports[0]["target"] == canonical, inc_imports + # Not an absolute-path-shaped ghost id (…_pkg_b would end "_b", but the + # pre-fix dangling form was the full path with the extension folded in). + assert not inc_imports[0]["target"].endswith("_py"), inc_imports + + root_slug = _make_id(str(tmp)) + for e in inc["edges"]: + assert root_slug not in str(e.get("target", "")), e + # The target_file hint is transient and must never ship. + assert "target_file" not in e, e + + +def test_incremental_md_reference_target_canonicalizes(tmp_path): + """#2211: a markdown [link](docs/setup.md) references edge must stamp + target_file so the #2169 remap canonicalizes its target on an incremental + extraction where the linked doc is NOT in the batch — instead of the + md->md reference dangling on an absolute-path-derived id and dropping.""" + from graphify.extract import extract, _make_id + + tmp = Path(os.path.realpath(tmp_path)) + docs = tmp / "docs" + docs.mkdir() + setup = docs / "setup.md" + setup.write_text("# Setup\nInstall the thing.\n", encoding="utf-8") + claude = tmp / "CLAUDE.md" + claude.write_text( + "# Overview\nSee [setup](docs/setup.md) for install steps.\n", + encoding="utf-8", + ) + + full = extract([claude, setup], cache_root=tmp) + full_refs = [ + e for e in full["edges"] + if e.get("relation") == "references" + and str(e.get("source_file", "")).endswith("CLAUDE.md") + ] + assert full_refs, full["edges"] + canonical = full_refs[0]["target"] + assert canonical == "docs_setup", canonical + + # Incremental: only the linking doc is in the batch. + inc = extract([claude], cache_root=tmp) + inc_refs = [ + e for e in inc["edges"] + if e.get("relation") == "references" + and str(e.get("source_file", "")).endswith("CLAUDE.md") + ] + assert inc_refs, inc["edges"] + assert inc_refs[0]["target"] == canonical, inc_refs + + root_slug = _make_id(str(tmp)) + for e in inc["edges"]: + assert root_slug not in str(e.get("target", "")), e + assert "target_file" not in e, e + + def test_update_prunes_a_removed_imports_edge(tmp_path): """#1521: when an import is deleted from a file, `graphify update` must prune the edge it produced — preserving it (keyed only on endpoint membership) left a diff --git a/tests/test_indirect_dispatch.py b/tests/test_indirect_dispatch.py index ee3d36bba..a1ceaed22 100644 --- a/tests/test_indirect_dispatch.py +++ b/tests/test_indirect_dispatch.py @@ -248,6 +248,31 @@ def test_cross_file_imported_callback_emits_indirect_call(tmp_path): assert e["confidence"] == "INFERRED" +def test_cross_file_class_ref_is_not_indirect_call(tmp_path): + """The cross-file resolver guard in extract.py must suppress indirect_call edges + whose target is a class imported from another module (a descriptor, not an + invocation), while a genuine imported function callback still emits its edge. + This exercises the _callable_class exclusion on the CROSS-FILE path, which the + intra-file test (test_class_ref_is_not_indirect_call) does not reach.""" + r, nid = _extract_dir(tmp_path, { + "models.py": ( + "class Widget:\n pass\n\n\n" + "def on_event(x):\n return x\n" + ), + "scheduler.py": ( + "from models import Widget, on_event\n\n\n" + "def schedule(pool, db, i):\n" + " db.get(Widget, i) # imported class as descriptor -> no edge\n" + " pool.submit(on_event) # imported fn callback -> indirect_call\n" + ), + }) + indirect = _rels(r, "indirect_call") + # imported class is never an indirect_call target across files + assert all(t != nid["Widget"] for _s, t in indirect) + # genuine imported function callback still emits its edge + assert (nid["schedule"], nid["on_event"]) in indirect + + def test_cross_file_affected_includes_importing_dispatcher(tmp_path): r, nid = _extract_dir(tmp_path, { "handlers.py": "def on_event(x):\n return x\n", @@ -504,3 +529,42 @@ def test_typescript_typed_params_and_arrow_consts(tmp_path): assert (nid["via"], nid["handler"]) in indirect assert (file_nid, nid["handler"]) in indirect assert (file_nid, nid["cb"]) in indirect + + +def test_class_ref_is_not_indirect_call(tmp_path): + """A class referenced BY NAME as a value is a descriptor, not an invocation, so it + must NOT emit an indirect_call edge — across all three symptom families of #2137: + ORM args (`select(Model)`, `db.get(Model, id)`), exception tuples + (`except (ErrorA, ErrorB)`), and string-literal getattr resolving to a same-named + class. A real function callback in the same file still emits its edge.""" + src = ( + "class ErrorA(Exception):\n pass\n" + "class ErrorB(Exception):\n pass\n" + "class KbArticle:\n pass\n" + "\n" + "def handler(x):\n return x\n" + "\n" + "def use_except():\n" + " try:\n pass\n" + " except (ErrorA, ErrorB) as e: # exception tuple -> classes, not calls\n" + " return e\n" + "\n" + "def use_getattr(run):\n" + ' return getattr(run, "KbArticle", 0) # literal resolving to a class\n' + "\n" + "def use_orm(db, i):\n" + " db.get(KbArticle, i) # class as descriptor\n" + " return select(KbArticle) # class as descriptor\n" + "\n" + "def register(pool):\n" + " pool.submit(handler) # real fn callback -> indirect_call\n" + ) + (tmp_path / "orm.py").write_text(src) + r = extract_python(tmp_path / "orm.py") + nid = {n["label"].rstrip("()"): n["id"] for n in r["nodes"]} + indirect = _rels(r, "indirect_call") + class_ids = {nid["ErrorA"], nid["ErrorB"], nid["KbArticle"]} + # no class is ever an indirect_call target (except-tuple, getattr, or ORM arg) + assert not any(t in class_ids for _, t in indirect) + # genuine function callback preserved + assert (nid["register"], nid["handler"]) in indirect diff --git a/tests/test_install.py b/tests/test_install.py index fc1a4e90b..8cea17b4c 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -359,6 +359,41 @@ def test_codebuddy_install_writes_hook(tmp_path): assert any("graphify" in str(h) for h in hooks) +def test_claude_hook_is_shell_agnostic(tmp_path): + # #522: the installed PreToolUse hooks must be plain exe invocations, not + # POSIX bash (which fails on Windows cmd.exe/PowerShell). + import json as _json + from graphify.__main__ import _install_claude_hook + _install_claude_hook(tmp_path) + hooks = _json.loads((tmp_path / ".claude" / "settings.json").read_text())["hooks"]["PreToolUse"] + matchers = {h["matcher"] for h in hooks} + assert {"Bash|Grep", "Read|Glob"} <= matchers # Grep in the search matcher: #1986 + for h in hooks: + cmd = h["hooks"][0]["command"] + for token in ("$(", "case ", "[ -f", "&&", "||", ";;", "echo '"): + assert token not in cmd, f"shell syntax {token!r} in {cmd!r}" + assert "graphify" in cmd and "hook-guard" in cmd + + +def test_claude_hook_install_idempotent_and_replaces_old_bash_hook(tmp_path): + import json as _json + from graphify.__main__ import _install_claude_hook + settings_path = tmp_path / ".claude" / "settings.json" + settings_path.parent.mkdir(parents=True) + # Pre-seed a legacy bash-style graphify hook (the thing #522 shipped before). + settings_path.write_text(_json.dumps({"hooks": {"PreToolUse": [ + {"matcher": "Bash", "hooks": [{"type": "command", + "command": "[ -f graphify-out/graph.json ] && echo '{...}' || true"}]}, + ]}}), encoding="utf-8") + _install_claude_hook(tmp_path) + _install_claude_hook(tmp_path) # second install must not duplicate + hooks = _json.loads(settings_path.read_text())["hooks"]["PreToolUse"] + graphify_hooks = [h for h in hooks if "graphify" in str(h)] + assert len(graphify_hooks) == 2, "exactly the Bash + Read|Glob guards, no dupes" + # the legacy bash payload must be gone + assert not any("[ -f graphify-out" in h["hooks"][0]["command"] for h in graphify_hooks) + + def test_codebuddy_install_idempotent(tmp_path): from graphify.__main__ import codebuddy_install codebuddy_install(tmp_path) @@ -588,6 +623,65 @@ def test_agents_uninstall_no_op_when_not_installed(tmp_path, capsys): assert "nothing to do" in out +def test_remove_marker_section_matches_exact_heading_only(tmp_path): + """#2062: the strip helper must match graphify's own `## graphify` heading + exactly, never a substring inside a user's `### graphify` H3.""" + from graphify.install import _remove_marker_section + m = "## graphify" + + # Only a user H3 mention -> no exact marker line -> None (file left untouched). + assert _remove_marker_section("# Doc\n\n### graphify\n\nmy notes\n", m) is None + # An inline/bullet mention is likewise not a section. + assert _remove_marker_section("see the ## graphify bullet\n", m) is None + + # A real H2 section alongside a user H3: remove only the H2 section. + content = "# Doc\n\n### graphify\n\nmy notes\n\n## graphify\n\ngraphify stuff\n" + out = _remove_marker_section(content, m) + assert out is not None + assert "### graphify" in out and "my notes" in out + assert not any(l.strip() == "## graphify" for l in out.splitlines()) + assert "graphify stuff" not in out + + # The section runs to the next H2 (not stopping at a `###` inside it). + c2 = "## graphify\n\nintro\n\n### sub\n\ninner\n\n## Keep\n\nkeep me\n" + out2 = _remove_marker_section(c2, m) + assert "## Keep" in out2 and "keep me" in out2 + assert "inner" not in out2 and "intro" not in out2 + + +def test_agents_uninstall_preserves_user_h3_graphify_heading(tmp_path): + """#2062 end-to-end: uninstall strips graphify's own H2 section but leaves a + user-authored `### graphify` H3 (and everything else) byte-intact.""" + agents_md = tmp_path / "AGENTS.md" + agents_md.write_text( + "# My rules\n\n" + "### graphify\n\n" + "My own notes on how I use graphify. Keep this.\n\n" + "## Other\n\nUnrelated content.\n" + ) + _agents_install(tmp_path, "codex") # appends a genuine `## graphify` H2 section + assert "## graphify" in agents_md.read_text() + + _agents_uninstall(tmp_path) + content = agents_md.read_text() + assert "### graphify" in content, "user's H3 heading was deleted (#2062)" + assert "My own notes on how I use graphify. Keep this." in content + assert "## Other" in content and "Unrelated content." in content + assert not any(l.strip() == "## graphify" for l in content.splitlines()) + + +def test_uninstall_untouched_when_only_user_h3_present(tmp_path, capsys): + """#2062: a file with only a user `### graphify` H3 (graphify never installed) + must be left byte-identical, not stripped.""" + agents_md = tmp_path / "AGENTS.md" + original = "# My rules\n\n### graphify\n\nHand-written. Do not touch.\n" + agents_md.write_text(original) + before = agents_md.read_bytes() + _agents_uninstall(tmp_path) + assert agents_md.read_bytes() == before + assert "nothing to do" in capsys.readouterr().out + + # --- OpenCode plugin tests --- @@ -1022,3 +1116,59 @@ def test_hermes_skill_destination_posix_uses_home(): with patch("graphify.__main__.platform.system", return_value="Linux"): dst = _platform_skill_destination("hermes", project=False) assert str(dst).endswith(".hermes/skills/graphify/SKILL.md"), dst + + +def _cli_dispatched_commands() -> set[str]: + """Subcommand names the CLI actually dispatches. + + `graphify`'s dispatcher is an `elif cmd == "..."` chain rather than a declarative + table, so the set is read back out of the source. Used to prove a hook command + written by an installer is not a stale/renamed subcommand (#2165). + """ + import re + from graphify import cli + + source = Path(cli.__file__).read_text(encoding="utf-8") + names = set(re.findall(r'cmd\s*==\s*"([a-z0-9][a-z0-9-]*)"', source)) + names |= { + m + for group in re.findall(r'cmd\s+in\s+\(([^)]*)\)', source) + for m in re.findall(r'"([a-z0-9][a-z0-9-]*)"', group) + } + return names + + +def test_codex_hook_command_is_a_real_cli_subcommand(tmp_path): + """#2165: the PreToolUse command in .codex/hooks.json must be a command the CLI + dispatches, so a renamed subcommand can never leave a permanently dead hook. + + `hook-check` is intentionally a no-op on Codex (Codex Desktop rejects + additionalContext on PreToolUse), but it must still be a *recognized* command -- + an unrecognized one exits non-zero and would break every Bash tool call. + """ + import json + + from graphify.install import _install_codex_hook + + _install_codex_hook(tmp_path) + hooks = json.loads((tmp_path / ".codex" / "hooks.json").read_text(encoding="utf-8")) + + entries = [ + h + for group in hooks["hooks"]["PreToolUse"] + for h in group["hooks"] + if "graphify" in h.get("command", "") + ] + assert entries, "codex install must register a graphify PreToolUse hook" + + dispatched = _cli_dispatched_commands() + assert "hook-check" in dispatched, "sanity: parser must find known commands" + + for entry in entries: + # command is " [args...]" + parts = entry["command"].split() + subcommand = parts[1] if len(parts) > 1 else "" + assert subcommand in dispatched, ( + f"codex hook registers {subcommand!r}, which the CLI does not dispatch " + f"(#2165). Known commands: {sorted(dispatched)}" + ) diff --git a/tests/test_install_strings.py b/tests/test_install_strings.py index 38bec1e02..7e65b768e 100644 --- a/tests/test_install_strings.py +++ b/tests/test_install_strings.py @@ -12,13 +12,13 @@ import json from graphify.__main__ import ( - _SETTINGS_HOOK, - _READ_SETTINGS_HOOK, + _SEARCH_NUDGE, + _READ_NUDGE, _skill_registration, _CLAUDE_MD_SECTION, _AGENTS_MD_SECTION, _GEMINI_MD_SECTION, - _GEMINI_HOOK, + _GEMINI_NUDGE_TEXT, _VSCODE_INSTRUCTIONS_SECTION, _ANTIGRAVITY_RULES, _KIRO_STEERING, @@ -32,12 +32,12 @@ # Hook constants are dicts/JSON; serialize them so we can do substring checks # against the actual payload text the assistant will receive. _INSTALL_TEXTS: dict[str, str] = { - "_SETTINGS_HOOK": json.dumps(_SETTINGS_HOOK), - "_READ_SETTINGS_HOOK": json.dumps(_READ_SETTINGS_HOOK), + "_SEARCH_NUDGE": _SEARCH_NUDGE, + "_READ_NUDGE": _READ_NUDGE, "_CLAUDE_MD_SECTION": _CLAUDE_MD_SECTION, "_AGENTS_MD_SECTION": _AGENTS_MD_SECTION, "_GEMINI_MD_SECTION": _GEMINI_MD_SECTION, - "_GEMINI_HOOK": json.dumps(_GEMINI_HOOK), + "_GEMINI_NUDGE_TEXT": _GEMINI_NUDGE_TEXT, "_VSCODE_INSTRUCTIONS_SECTION": _VSCODE_INSTRUCTIONS_SECTION, "_ANTIGRAVITY_RULES": _ANTIGRAVITY_RULES, "_KIRO_STEERING": _KIRO_STEERING, diff --git a/tests/test_install_upgrade.py b/tests/test_install_upgrade.py index 13e7c55a5..e85add452 100644 --- a/tests/test_install_upgrade.py +++ b/tests/test_install_upgrade.py @@ -145,9 +145,14 @@ def test_claude_install_upgrades_stale_hook_payload(tmp_path, monkeypatch): assert _OLD_HOOK_PAYLOAD_SNIPPET not in new_settings_text, ( "stale hook payload survived upgrade" ) - assert "graphify query" in new_settings_text, ( - "new hook payload should route to `graphify query`" + # Since #522 the nudge text lives in the `graphify hook-guard` subcommand, not + # inline in settings.json (so the command parses on Windows). The upgraded hook + # must therefore route to that shell-agnostic subcommand, and the old bash + # pipeline must be gone. + assert "hook-guard" in new_settings_text, ( + "new hook payload should route to the `graphify hook-guard` subcommand" ) + assert "case x in" not in new_settings_text, "stale bash pipeline survived upgrade" def test_agents_install_upgrades_stale_section(tmp_path, monkeypatch): diff --git a/tests/test_java_member_calls.py b/tests/test_java_member_calls.py new file mode 100644 index 000000000..2f4a669b6 --- /dev/null +++ b/tests/test_java_member_calls.py @@ -0,0 +1,278 @@ +"""Java receiver-typed member-call resolution. + +Java ``method_invocation`` nodes carry both the method name and its receiver, +but the generic extractor currently resolves only by the bare method name. A +typed receiver must select the method owned by its declared type; unresolved or +ambiguous receivers must stay unlinked rather than creating a false call edge. +""" +from __future__ import annotations + +from pathlib import Path + +from graphify.extract import extract + + +def _calls(tmp_path: Path, files: dict[str, str]): + paths = [] + for name, body in files.items(): + path = tmp_path / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body, encoding="utf-8") + paths.append(path) + result = extract(paths, cache_root=tmp_path / "graphify-out") + calls = { + (edge["source"], edge["target"]) + for edge in result["edges"] + if edge.get("relation") == "calls" + } + return calls, result + + +def _find(result: dict, label: str, id_contains: str) -> str: + return next( + node["id"] + for node in result["nodes"] + if node.get("label") == label and id_contains in node["id"] + ) + + +_AMBIGUOUS_METHODS = { + "Services.java": ( + "class PaymentGateway { static void ping() {} void charge() {} }\n" + "class AuditLog { static void ping() {} void charge() {} }\n" + ), +} + + +def test_explicit_type_receiver_resolves_to_owned_method(tmp_path: Path): + calls, result = _calls(tmp_path, { + **_AMBIGUOUS_METHODS, + "Checkout.java": ( + "class Checkout { void run() { PaymentGateway.ping(); } }\n" + ), + }) + + run = _find(result, ".run()", "checkout") + gateway_ping = _find(result, ".ping()", "paymentgateway") + audit_ping = _find(result, ".ping()", "auditlog") + assert (run, gateway_ping) in calls + assert (run, audit_ping) not in calls + + +def test_field_receiver_resolves_to_declared_type(tmp_path: Path): + calls, result = _calls(tmp_path, { + **_AMBIGUOUS_METHODS, + "Checkout.java": ( + "class Checkout {\n" + " void run() { gateway.charge(); }\n" + " PaymentGateway gateway;\n" + "}\n" + ), + }) + + run = _find(result, ".run()", "checkout") + gateway_charge = _find(result, ".charge()", "paymentgateway") + audit_charge = _find(result, ".charge()", "auditlog") + assert (run, gateway_charge) in calls + assert (run, audit_charge) not in calls + + +def test_this_field_receiver_resolves_to_declared_type(tmp_path: Path): + calls, result = _calls(tmp_path, { + **_AMBIGUOUS_METHODS, + "Checkout.java": ( + "class Checkout {\n" + " PaymentGateway gateway;\n" + " void run() { this.gateway.charge(); }\n" + "}\n" + ), + }) + + run = _find(result, ".run()", "checkout") + gateway_charge = _find(result, ".charge()", "paymentgateway") + assert (run, gateway_charge) in calls + + +def test_this_field_uses_field_type_when_parameter_shadows_name(tmp_path: Path): + calls, result = _calls(tmp_path, { + **_AMBIGUOUS_METHODS, + "Checkout.java": ( + "class Checkout {\n" + " PaymentGateway service;\n" + " void run(AuditLog service) {\n" + " service.charge();\n" + " this.service.charge();\n" + " }\n" + "}\n" + ), + }) + + run = _find(result, ".run()", "checkout") + gateway_charge = _find(result, ".charge()", "paymentgateway") + audit_charge = _find(result, ".charge()", "auditlog") + assert (run, gateway_charge) in calls + assert (run, audit_charge) in calls + + +def test_parameter_and_local_receivers_resolve_per_method(tmp_path: Path): + calls, result = _calls(tmp_path, { + **_AMBIGUOUS_METHODS, + "Checkout.java": ( + "class Checkout {\n" + " void fromParameter(PaymentGateway service) { service.charge(); }\n" + " void fromLocal() { AuditLog service = new AuditLog(); service.charge(); }\n" + "}\n" + ), + }) + + from_parameter = _find(result, ".fromParameter()", "checkout") + from_local = _find(result, ".fromLocal()", "checkout") + gateway_charge = _find(result, ".charge()", "paymentgateway") + audit_charge = _find(result, ".charge()", "auditlog") + assert (from_parameter, gateway_charge) in calls + assert (from_parameter, audit_charge) not in calls + assert (from_local, audit_charge) in calls + assert (from_local, gateway_charge) not in calls + + +def test_nested_receiver_bindings_do_not_escape_their_scope(tmp_path: Path): + calls, result = _calls(tmp_path, { + **_AMBIGUOUS_METHODS, + "Checkout.java": ( + "class Checkout {\n" + " PaymentGateway service;\n" + " void blockLocal() {\n" + " service.charge();\n" + " { AuditLog service = null; service.charge(); }\n" + " }\n" + " void anonymousClass() {\n" + " new Object() { void nested() { AuditLog service = null; } };\n" + " service.charge();\n" + " }\n" + "}\n" + ), + }) + + block_local = _find(result, ".blockLocal()", "checkout") + anonymous_class = _find(result, ".anonymousClass()", "checkout") + gateway_charge = _find(result, ".charge()", "paymentgateway") + audit_charge = _find(result, ".charge()", "auditlog") + assert not any(source == block_local and "charge" in target + for source, target in calls) + assert (anonymous_class, gateway_charge) in calls + assert (anonymous_class, audit_charge) not in calls + + +def test_lambda_shadowing_does_not_reuse_enclosing_receiver_type(tmp_path: Path): + calls, result = _calls(tmp_path, { + **_AMBIGUOUS_METHODS, + "Checkout.java": ( + "class Checkout {\n" + " PaymentGateway service;\n" + " void captured() {\n" + " Runnable task = () -> service.charge();\n" + " }\n" + " void shadowed() {\n" + " java.util.function.Consumer task =\n" + " service -> service.charge();\n" + " }\n" + " void parenthesized() {\n" + " java.util.function.Consumer task =\n" + " (service) -> service.charge();\n" + " }\n" + " void typed() {\n" + " java.util.function.Consumer task =\n" + " (AuditLog service) -> service.charge();\n" + " }\n" + " void sameType() {\n" + " java.util.function.Consumer task =\n" + " (PaymentGateway service) -> service.charge();\n" + " }\n" + "}\n" + ), + }) + + captured = _find(result, ".captured()", "checkout") + same_type = _find(result, ".sameType()", "checkout") + shadowed_callers = { + _find(result, f".{name}()", "checkout") + for name in ("shadowed", "parenthesized", "typed") + } + gateway_charge = _find(result, ".charge()", "paymentgateway") + assert (captured, gateway_charge) in calls + assert (same_type, gateway_charge) in calls + assert not any(source in shadowed_callers and "charge" in target + for source, target in calls) + + +def test_overloaded_callers_keep_body_scoped_receiver_types(tmp_path: Path): + calls, result = _calls(tmp_path, { + **_AMBIGUOUS_METHODS, + "Checkout.java": ( + "class Checkout {\n" + " void run(int value) { PaymentGateway service = null; service.charge(); }\n" + " void run(String value) { AuditLog service = null; service.charge(); }\n" + "}\n" + ), + }) + + run = _find(result, ".run()", "checkout") + gateway_charge = _find(result, ".charge()", "paymentgateway") + audit_charge = _find(result, ".charge()", "auditlog") + assert (run, gateway_charge) in calls + assert (run, audit_charge) in calls + + +def test_ambiguous_receiver_type_emits_no_edge(tmp_path: Path): + calls, result = _calls(tmp_path, { + "a/Gateway.java": "package a; public class Gateway { public void send() {} }\n", + "b/Gateway.java": "package b; public class Gateway { public void send() {} }\n", + "Caller.java": ( + "class Caller { void run(Gateway gateway) { gateway.send(); } }\n" + ), + }) + + run = _find(result, ".run()", "caller") + send_targets = { + target + for source, target in calls + if source == run and "send" in target + } + assert send_targets == set() + + +def test_inherited_field_and_chained_receiver_are_deferred(tmp_path: Path): + calls, result = _calls(tmp_path, { + "Services.java": ( + "class Gateway { void charge() {} Gateway create() { return this; } }\n" + "class Base { Gateway gateway; }\n" + "class Checkout extends Base {\n" + " Gateway factory;\n" + " void inherited() { this.gateway.charge(); }\n" + " void chained() { factory.create().charge(); }\n" + "}\n" + ), + }) + + inherited = _find(result, ".inherited()", "checkout") + chained = _find(result, ".chained()", "checkout") + assert not any(source in {inherited, chained} and "charge" in target + for source, target in calls) + + +def test_unqualified_call_still_resolves(tmp_path: Path): + calls, result = _calls(tmp_path, { + "Checkout.java": ( + "class Checkout {\n" + " void run() { helper(); this.other(); }\n" + " void helper() {}\n" + " void other() {}\n" + "}\n" + ), + }) + + run = _find(result, ".run()", "checkout") + helper = _find(result, ".helper()", "checkout") + other = _find(result, ".other()", "checkout") + assert (run, helper) in calls + assert (run, other) in calls diff --git a/tests/test_java_type_resolution.py b/tests/test_java_type_resolution.py index 35229776e..77297498a 100644 --- a/tests/test_java_type_resolution.py +++ b/tests/test_java_type_resolution.py @@ -77,6 +77,58 @@ def test_java_ambiguous_implements_disambiguated_by_import(tmp_path: Path): assert "com/b/handler" not in tgt["source_file"] +def test_java_ambiguous_reference_disambiguated_by_import(tmp_path: Path): + # #1744: two classes with the SAME simple name in different modules/packages. + # Both survive as distinct path-scoped nodes, but a cross-module field/type + # `references` edge used to dangle on a sourceless phantom stub (the + # implements/inherits case was handled, references was not). The importing + # file's `import` must re-point the reference to the right class and leave no + # orphan phantom. + payment = _write( + tmp_path / "payment/src/com/example/payment/FinancialEntryValidator.java", + "package com.example.payment;\n" + "public class FinancialEntryValidator {\n" + " public boolean validateCurrency(String c) { return c.length() == 3; }\n" + "}\n", + ) + core = _write( + tmp_path / "core/src/com/example/core/FinancialEntryValidator.java", + "package com.example.core;\n" + "public class FinancialEntryValidator {\n" + " public void auditEntry(String id) {}\n" + "}\n", + ) + consumer = _write( + tmp_path / "app/src/com/example/app/PaymentService.java", + "package com.example.app;\n" + "import com.example.payment.FinancialEntryValidator;\n" + "public class PaymentService {\n" + " private FinancialEntryValidator validator = new FinancialEntryValidator();\n" + "}\n", + ) + result = extract([payment, core, consumer], cache_root=tmp_path) + + # Both real classes survive (path-scoped ids); no sourceless phantom remains. + fev = [n for n in result["nodes"] if n.get("label") == "FinancialEntryValidator"] + reals = [n for n in fev if n.get("source_file")] + phantoms = [n for n in fev if not n.get("source_file")] + assert len(reals) == 2, f"expected both real classes, got {[n.get('source_file') for n in fev]}" + assert not phantoms, f"orphan phantom node(s) remain: {[n['id'] for n in phantoms]}" + + # The reference must resolve to the IMPORTED (payment) class, not core. + refs = [ + e for e in result["edges"] + if e["relation"] == "references" + and (_node_by_id(result, e["target"]) or {}).get("label") == "FinancialEntryValidator" + ] + assert refs, "expected a references edge to FinancialEntryValidator" + for e in refs: + tgt = _node_by_id(result, e["target"]) + assert tgt is not None and tgt.get("source_file") + assert "payment/" in tgt["source_file"] + assert "core/" not in tgt["source_file"] + + def test_java_implements_edge_survives_build(tmp_path: Path): # #1318: the re-pointed edge must connect real nodes after graph assembly, # so the interface is not classified as an isolated community. diff --git a/tests/test_js_import_resolution.py b/tests/test_js_import_resolution.py index b8314a971..49452a98c 100644 --- a/tests/test_js_import_resolution.py +++ b/tests/test_js_import_resolution.py @@ -1212,15 +1212,378 @@ def test_alias_import_edge_resolves_with_relative_input_paths(tmp_path, monkeypa # that exists in the graph), not an orphan keyed by an absolute prefix. assert _has_edge(result, "src/components/Button.tsx", "src/lib/utils.ts") assert target_id in node_ids - import_targets = { + import_targets = [ e["target"] for e in result["edges"] if e["relation"] == "imports_from" and e["source"] == _file_node_id(Path("src/components/Button.tsx")) - } - assert target_id in import_targets + ] + assert import_targets == [target_id] # No surviving edge target may carry an absolute-path prefix from tmp_path. abs_prefix = _file_node_id(Path("src/lib/utils.ts").resolve()) assert all(not t.startswith(abs_prefix + "_") and t != abs_prefix for t in import_targets) # The named-symbol edge to formatDate must resolve to the real symbol node too. assert _has_symbol_edge(result, "src/components/Button.tsx", "src/lib/utils.ts", "formatDate") + symbol_target = _make_id(_file_stem(Path("src/lib/utils.ts")), "formatDate") + named_imports = [ + edge + for edge in result["edges"] + if edge["source"] == _file_node_id(Path("src/components/Button.tsx")) + and edge["relation"] == "imports" + and edge["source_location"] == "L1" + ] + assert [edge["target"] for edge in named_imports] == [symbol_target] + assert all( + edge["source"] in node_ids and edge["target"] in node_ids + for edge in result["edges"] + if edge["relation"] in ("imports", "imports_from") + ) + + +def test_alias_import_symbol_resolves_from_parent_working_directory(tmp_path, monkeypatch): + project = tmp_path / "project" + _write( + project / "tsconfig.json", + json.dumps({"compilerOptions": {"baseUrl": ".", "paths": {"@/*": ["src/*"]}}}), + ) + _write(project / "src/lib/utils.ts", "export function formatDate(d) { return d }\n") + _write( + project / "src/components/Button.tsx", + "import { formatDate } from '@/lib/utils'\n", + ) + + monkeypatch.chdir(tmp_path) + result = extract( + [Path("project/src/lib/utils.ts"), Path("project/src/components/Button.tsx")], + cache_root=Path("project"), + ) + + node_ids = {node["id"] for node in result["nodes"]} + source_id = _file_node_id(Path("src/components/Button.tsx")) + symbol_target = _make_id(_file_stem(Path("src/lib/utils.ts")), "formatDate") + named_imports = [ + edge + for edge in result["edges"] + if edge["source"] == source_id and edge["relation"] == "imports" + ] + + assert [edge["target"] for edge in named_imports] == [symbol_target] + assert all(edge["source"] in node_ids and edge["target"] in node_ids for edge in named_imports) + + +@pytest.mark.parametrize( + "statement", + [ + "export { formatDate } from '@/lib/utils'\n", + "export { formatDate as displayDate } from '@/lib/utils'\n", + ], +) +def test_alias_reexport_symbol_resolves_with_relative_input_paths( + tmp_path, monkeypatch, statement +): + _write( + tmp_path / "tsconfig.json", + json.dumps({"compilerOptions": {"baseUrl": ".", "paths": {"@/*": ["src/*"]}}}), + ) + _write(tmp_path / "src/lib/utils.ts", "export function formatDate() { return 'ok' }\n") + _write(tmp_path / "src/lib/index.ts", statement) + + monkeypatch.chdir(tmp_path) + target = Path("src/lib/utils.ts") + barrel = Path("src/lib/index.ts") + result = extract([target, barrel], cache_root=Path(".")) + + node_ids = {node["id"] for node in result["nodes"]} + file_target = _file_node_id(target) + symbol_target = _make_id(_file_stem(target), "formatDate") + reexports = [ + edge + for edge in result["edges"] + if edge["source"] == _file_node_id(barrel) + and edge["relation"] == "re_exports" + ] + + assert sorted(edge["target"] for edge in reexports) == sorted( + [file_target, symbol_target] + ) + assert all(edge["target"] in node_ids for edge in reexports) + absolute_prefix = _file_node_id(target.resolve()) + assert all(not edge["target"].startswith(absolute_prefix + "_") for edge in reexports) + + +def test_alias_reexport_symbol_resolves_from_parent_working_directory(tmp_path, monkeypatch): + project = tmp_path / "project" + _write( + project / "tsconfig.json", + json.dumps({"compilerOptions": {"baseUrl": ".", "paths": {"@/*": ["src/*"]}}}), + ) + _write(project / "src/lib/utils.ts", "export function formatDate() { return 'ok' }\n") + _write(project / "src/lib/index.ts", "export { formatDate } from '@/lib/utils'\n") + + monkeypatch.chdir(tmp_path) + target = Path("project/src/lib/utils.ts") + barrel = Path("project/src/lib/index.ts") + result = extract([target, barrel], cache_root=Path("project")) + + node_ids = {node["id"] for node in result["nodes"]} + source = _file_node_id(Path("src/lib/index.ts")) + symbol_target = _make_id(_file_stem(Path("src/lib/utils.ts")), "formatDate") + symbol_reexports = [ + edge + for edge in result["edges"] + if edge["source"] == source + and edge["relation"] == "re_exports" + and edge["target"] != _file_node_id(Path("src/lib/utils.ts")) + ] + + assert [edge["target"] for edge in symbol_reexports] == [symbol_target] + assert all(edge["target"] in node_ids for edge in symbol_reexports) + + +def test_alias_reexport_does_not_rewrite_an_owned_symbol_id(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + target = Path("src/lib/utils.ts") + absolute_prefix = _file_node_id(target.resolve()) + mirror = Path(f"{absolute_prefix}.ts") + barrel = Path("src/lib/index.ts") + + _write( + Path("tsconfig.json"), + json.dumps({"compilerOptions": {"baseUrl": ".", "paths": {"@/*": ["src/*"]}}}), + ) + _write(target, "export function formatDate() { return 'target' }\n") + _write(mirror, "export function formatDate() { return 'mirror' }\n") + _write(barrel, "export { formatDate } from '@/lib/utils'\n") + + result = extract([target, mirror, barrel], cache_root=Path(".")) + + node_ids = {node["id"] for node in result["nodes"]} + owned_target = _make_id(_file_stem(mirror), "formatDate") + symbol_reexports = [ + edge + for edge in result["edges"] + if edge["source"] == _file_node_id(barrel) + and edge["relation"] == "re_exports" + and edge["target"] != _file_node_id(target) + ] + + assert [edge["target"] for edge in symbol_reexports] == [owned_target] + assert owned_target in node_ids + + +def test_alias_import_does_not_remap_an_owned_symbol_id(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + target = Path("src/lib/utils.ts") + absolute_prefix = _file_node_id(target.resolve()) + mirror = Path(f"{absolute_prefix}.ts") + button = Path("src/components/Button.tsx") + mirror_user = Path("src/components/Mirror.tsx") + + _write( + Path("tsconfig.json"), + json.dumps({"compilerOptions": {"baseUrl": ".", "paths": {"@/*": ["src/*"]}}}), + ) + _write(target, "export function formatDate(d) { return d }\n") + _write(mirror, "export function formatDate(d) { return 999 }\n") + _write( + button, + "import { formatDate } from '@/lib/utils'\nexport const a = formatDate(1)\n", + ) + _write( + mirror_user, + f"import {{ formatDate }} from '../../{mirror.stem}'\nexport const b = formatDate(2)\n", + ) + + result = extract( + [target, mirror, button, mirror_user], + cache_root=Path("."), + ) + + node_ids = {node["id"] for node in result["nodes"]} + symbols = { + node["source_file"]: node["id"] + for node in result["nodes"] + if node.get("label") == "formatDate()" + } + target_symbol = _make_id(_file_stem(target), "formatDate") + mirror_symbol = _make_id(_file_stem(mirror), "formatDate") + assert symbols[str(target)] == target_symbol + assert symbols[str(mirror)] == mirror_symbol + + imports = [ + edge + for edge in result["edges"] + if edge["relation"] == "imports" and edge["source_location"] == "L1" + ] + by_source: dict[str, list[str]] = {} + for edge in imports: + by_source.setdefault(edge["source_file"], []).append(edge["target"]) + assert by_source[str(button)] == [target_symbol] + assert by_source[str(mirror_user)] == [mirror_symbol] + assert all(edge["source"] in node_ids and edge["target"] in node_ids for edge in imports) + + +def test_alias_import_preserves_owned_same_line_symbol_edge(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + target = Path("src/lib/utils.ts") + absolute_prefix = _file_node_id(target.resolve()) + mirror = Path(f"{absolute_prefix}.ts") + importer = Path("src/components/Both.tsx") + + _write( + Path("tsconfig.json"), + json.dumps({"compilerOptions": {"baseUrl": ".", "paths": {"@/*": ["src/*"]}}}), + ) + _write(target, "export function formatDate(d) { return d }\n") + _write(mirror, "export function formatDate(d) { return 999 }\n") + _write( + importer, + f"import {{ formatDate as a }} from '@/lib/utils'; " + f"import {{ formatDate as b }} from '../../{mirror.stem}';\n" + "export const value = a(1) + b(2)\n", + ) + + result = extract([target, mirror, importer], cache_root=Path(".")) + + node_ids = {node["id"] for node in result["nodes"]} + target_symbol = _make_id(_file_stem(target), "formatDate") + mirror_symbol = _make_id(_file_stem(mirror), "formatDate") + imports = [ + edge + for edge in result["edges"] + if edge["source"] == _file_node_id(importer) + and edge["relation"] == "imports" + and edge["source_location"] == "L1" + ] + + assert sorted(edge["target"] for edge in imports) == sorted([target_symbol, mirror_symbol]) + assert all(edge["source"] in node_ids and edge["target"] in node_ids for edge in imports) + + +# --- #1983 (follow-up): alias re-exports THROUGH a barrel --------------------- +# The candidates rewrite learns old->canonical symbol forms only from symbols a +# file DEFINES. A barrel defines nothing, so a re-export/import that resolves to +# the barrel synthesizes an absolute-prefixed target no rewrite ever learns: +# the checkout path leaks into the id and the edge dangles. + +def _barrel_fixture(tmp_path): + _write( + tmp_path / "tsconfig.json", + json.dumps({"compilerOptions": {"baseUrl": ".", "paths": {"@/*": ["src/*"]}}}), + ) + _write(tmp_path / "src/lib/utils.ts", "export function formatDate() { return 'ok' }\n") + _write(tmp_path / "src/lib/index.ts", "export { formatDate } from '@/lib/utils'\n") + + +def test_alias_reexport_through_barrel_resolves_to_defining_symbol(tmp_path, monkeypatch): + _barrel_fixture(tmp_path) + _write(tmp_path / "src/barrel2.ts", "export { formatDate } from '@/lib'\n") + + monkeypatch.chdir(tmp_path) + files = sorted(Path("src").rglob("*.ts")) + result = extract(files, cache_root=Path(".")) + + node_ids = {node["id"] for node in result["nodes"]} + defining_symbol = _make_id(_file_stem(Path("src/lib/utils.ts")), "formatDate") + barrel_reexports = [ + edge + for edge in result["edges"] + if edge["source"] == _file_node_id(Path("src/barrel2.ts")) + and edge["relation"] == "re_exports" + and edge["target"] != _file_node_id(Path("src/lib/index.ts")) + ] + + assert [edge["target"] for edge in barrel_reexports] == [defining_symbol] + assert all(edge["target"] in node_ids for edge in barrel_reexports) + + +def test_alias_reexport_two_hop_barrel_chain_resolves(tmp_path, monkeypatch): + _barrel_fixture(tmp_path) + _write(tmp_path / "src/barrel2.ts", "export { formatDate } from '@/lib'\n") + _write(tmp_path / "src/barrel3.ts", "export { formatDate } from '@/barrel2'\n") + + monkeypatch.chdir(tmp_path) + files = sorted(Path("src").rglob("*.ts")) + result = extract(files, cache_root=Path(".")) + + node_ids = {node["id"] for node in result["nodes"]} + defining_symbol = _make_id(_file_stem(Path("src/lib/utils.ts")), "formatDate") + barrel3_reexports = [ + edge + for edge in result["edges"] + if edge["source"] == _file_node_id(Path("src/barrel3.ts")) + and edge["relation"] == "re_exports" + and edge["target"] != _file_node_id(Path("src/barrel2.ts")) + ] + + assert [edge["target"] for edge in barrel3_reexports] == [defining_symbol] + assert all(edge["target"] in node_ids for edge in barrel3_reexports) + + +def test_no_symbol_edge_target_contains_checkout_prefix(tmp_path, monkeypatch): + """No re_exports/imports target may embed the absolute checkout path — + the core complaint of #1983, which barrels still triggered after the + single-hop fix.""" + _barrel_fixture(tmp_path) + _write(tmp_path / "src/barrel2.ts", "export { formatDate } from '@/lib'\n") + _write( + tmp_path / "src/consumer.ts", + "import { formatDate } from '@/lib'\nexport function useIt() { return formatDate() }\n", + ) + + monkeypatch.chdir(tmp_path) + files = sorted(Path("src").rglob("*.ts")) + result = extract(files, cache_root=Path(".")) + + abs_prefix = _make_id(str(Path("src").resolve().parent)) + offenders = [ + (edge["relation"], edge["target"]) + for edge in result["edges"] + if edge.get("relation") in ("re_exports", "imports") + and str(edge.get("target", "")).startswith(abs_prefix) + ] + assert offenders == [], f"checkout path leaked into edge targets: {offenders}" + + +def test_ambiguous_barrel_reexport_chain_does_not_guess(tmp_path, monkeypatch): + """When a barrel re-exports the SAME local name from two different modules, + the barrel-chain resolver must NOT collapse an importer's edge onto one of + them by last-write-wins (#2034 follow-up). With the ambiguity guard the chain + leaves the import unresolved at the barrel symbol (dangling, dropped at build) + rather than fabricating a specific target; without it the chain repoints to + whichever module was learned last.""" + _write( + tmp_path / "tsconfig.json", + json.dumps({"compilerOptions": {"baseUrl": ".", "paths": {"@/*": ["src/*"]}}}), + ) + _write(tmp_path / "src/lib/a.ts", "export function dup() { return 'a' }\n") + _write(tmp_path / "src/lib/b.ts", "export function dup() { return 'b' }\n") + _write(tmp_path / "src/lib/index.ts", + "export { dup } from '@/lib/a'\nexport { dup } from '@/lib/b'\n") + _write(tmp_path / "src/consumer.ts", + "import { dup } from '@/lib'\nexport function useIt() { return dup() }\n") + + monkeypatch.chdir(tmp_path) + result = extract(sorted(Path("src").rglob("*.ts")), cache_root=Path(".")) + + barrel_sym = _make_id(_file_stem(Path("src/lib/index.ts")), "dup") + consumer = _file_node_id(Path("src/consumer.ts")) + consumer_imports = [ + e for e in result["edges"] + if e.get("source") == consumer and e.get("relation") == "imports" + ] + # The chain-produced import edge stays at the barrel symbol (unresolved) — + # proof the chain refused to guess. Without the fix it would be repointed to + # src_lib_a_dup / src_lib_b_dup (last-write-wins), so this edge would vanish. + assert any(e.get("target") == barrel_sym for e in consumer_imports), ( + f"ambiguous barrel import was chain-resolved instead of left unresolved: " + f"{[e.get('target') for e in consumer_imports]}" + ) + # Both legitimate barrel re-exports still resolve to their own module. + barrel = _file_node_id(Path("src/lib/index.ts")) + reexport_targets = { + e.get("target") for e in result["edges"] + if e.get("source") == barrel and e.get("relation") == "re_exports" + } + assert _make_id(_file_stem(Path("src/lib/a.ts")), "dup") in reexport_targets + assert _make_id(_file_stem(Path("src/lib/b.ts")), "dup") in reexport_targets diff --git a/tests/test_jsconfig_baseurl.py b/tests/test_jsconfig_baseurl.py new file mode 100644 index 000000000..c99d07950 --- /dev/null +++ b/tests/test_jsconfig_baseurl.py @@ -0,0 +1,193 @@ +"""Regression tests: jsconfig.json / baseUrl module resolution (#2153). + +`compilerOptions.baseUrl` was only ever used as the base that `paths` aliases +resolve against, so a config declaring `baseUrl` and NO `paths` produced an +empty alias map and every non-relative import died. In a Rails/webpacker +project (`jsconfig.json` with `"baseUrl": "app/javascript"`) that orphaned every +module: `import Widget from 'mods/Widget.js'` emitted no edge, so `affected` +answered nothing. `jsconfig.json` was also never probed at all — only +`tsconfig.json` — even though json_config.py already indexes it. + +baseUrl is now a resolution root of last resort: tried only after every declared +alias fails to match, so existing `paths` precedence (#1269, #927, #1531) is +untouched. +""" +from pathlib import Path + +from graphify.extract import _make_id, extract + + +def _write(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def _targets(result: dict) -> set[str]: + return {e["target"] for e in result["edges"]} + + +def _rails_tree(tmp_path: Path, config_name: str, importer_body: str, + base_url: str = "app/javascript", paths: str = "") -> Path: + """A webpacker-shaped project: config at the root, modules under baseUrl.""" + paths_frag = f',\n "paths": {{ {paths} }}' if paths else "" + _write(tmp_path / config_name, + '{\n "compilerOptions": {\n' + f' "baseUrl": "{base_url}"{paths_frag}\n' + ' }\n}\n') + _write(tmp_path / base_url / "mods" / "Widget.js", + "export default function Widget() {}\n") + return _write(tmp_path / base_url / "packs" / "dashboard.js", importer_body) + + +def _widget(tmp_path: Path, base_url: str = "app/javascript") -> str: + # The resolved import target is canonicalized to the root-relative file-node + # id (the same id Widget.js gets as a node), not the absolute-path form + # (#2169 canonicalizes cross-file edge targets). + return _cid(tmp_path, tmp_path / base_url / "mods" / "Widget.js") + + +def _cid(tmp_path: Path, abs_path: Path) -> str: + """Canonical root-relative file-node id of a cross-file import target (#2169).""" + return _make_id(str(Path(abs_path).relative_to(tmp_path).with_suffix(""))) + + +def test_jsconfig_baseurl_static_import_resolves(tmp_path): + # The issue repro: jsconfig.json, baseUrl, NO paths. + f = _rails_tree(tmp_path, "jsconfig.json", + "import Widget from 'mods/Widget.js';\n" + "export default Widget;\n") + r = extract([f], cache_root=tmp_path) + assert _widget(tmp_path) in _targets(r) + + +def test_jsconfig_baseurl_dynamic_import_resolves(tmp_path): + # The issue states static AND dynamic are both affected. + f = _rails_tree(tmp_path, "jsconfig.json", + "export async function load() {\n" + " return import('mods/Widget.js');\n" + "}\n") + r = extract([f], cache_root=tmp_path) + assert _widget(tmp_path) in _targets(r) + + +def test_jsconfig_baseurl_extensionless_specifier_resolves(tmp_path): + f = _rails_tree(tmp_path, "jsconfig.json", + "import Widget from 'mods/Widget';\n" + "export default Widget;\n") + r = extract([f], cache_root=tmp_path) + assert _widget(tmp_path) in _targets(r) + + +def test_tsconfig_baseurl_without_paths_also_resolves(tmp_path): + # Same defect applies to tsconfig.json, not just jsconfig.json. + f = _rails_tree(tmp_path, "tsconfig.json", + "import Widget from 'mods/Widget.js';\n" + "export default Widget;\n") + r = extract([f], cache_root=tmp_path) + assert _widget(tmp_path) in _targets(r) + + +def test_declared_paths_alias_still_wins_over_baseurl(tmp_path): + # Guards #1269: a declared alias must not be shadowed by the baseUrl + # fallback. "@mods/Widget.js" must resolve via paths to real/Widget.js, + # NOT to /@mods/Widget.js. + _write(tmp_path / "jsconfig.json", + '{\n "compilerOptions": {\n' + ' "baseUrl": "app/javascript",\n' + ' "paths": { "@mods/*": ["real/*"] }\n' + ' }\n}\n') + # paths targets are relative to baseUrl (#1269), so "real/*" is + # /real/*. + real = _write(tmp_path / "app/javascript" / "real" / "Widget.js", + "export default 1;\n") + _write(tmp_path / "app/javascript" / "@mods" / "Widget.js", "export default 2;\n") + f = _write(tmp_path / "app/javascript" / "packs" / "d.js", + "import W from '@mods/Widget.js';\nexport default W;\n") + r = extract([f], cache_root=tmp_path) + targets = _targets(r) + assert _cid(tmp_path, real) in targets + assert _cid(tmp_path, tmp_path / "app/javascript" / "@mods" / "Widget.js") not in targets + + +def test_declared_directory_prefix_alias_still_wins(tmp_path): + # A non-wildcard alias used as a directory prefix scores WORSE than a + # wildcard in _match_tsconfig_alias, so a naive implicit "*" alias would + # shadow it. The fallback must not. + _write(tmp_path / "jsconfig.json", + '{\n "compilerOptions": {\n' + ' "baseUrl": "app/javascript",\n' + ' "paths": { "@lib": ["src/lib"] }\n' + ' }\n}\n') + real = _write(tmp_path / "app/javascript" / "src" / "lib" / "util.js", + "export default 1;\n") + _write(tmp_path / "app/javascript" / "@lib" / "util.js", "export default 2;\n") + f = _write(tmp_path / "app/javascript" / "packs" / "p.js", + "import u from '@lib/util.js';\nexport default u;\n") + r = extract([f], cache_root=tmp_path) + assert _cid(tmp_path, real) in _targets(r) + + +def test_tsconfig_paths_alias_unchanged(tmp_path): + # Pre-existing #1269 behavior, green before and after this change: a + # tsconfig paths alias resolves relative to baseUrl. + _write(tmp_path / "tsconfig.json", + '{\n "compilerOptions": {\n' + ' "baseUrl": "./src",\n' + ' "paths": { "@services/*": ["services/*"] }\n' + ' }\n}\n') + svc = _write(tmp_path / "src" / "services" / "api.ts", "export const a = 1;\n") + f = _write(tmp_path / "src" / "app" / "main.ts", + "import { a } from '@services/api';\nexport default a;\n") + r = extract([f], cache_root=tmp_path) + assert _cid(tmp_path, svc) in _targets(r) + + +def test_external_package_not_fabricated_under_baseurl(tmp_path): + # baseUrl must not turn a real npm import into a phantom /react + # edge; the fallback only returns paths that exist on disk. + _write(tmp_path / "jsconfig.json", + '{\n "compilerOptions": { "baseUrl": "app/javascript" }\n}\n') + f = _write(tmp_path / "app/javascript" / "packs" / "p.js", + "import React from 'react';\nexport default React;\n") + r = extract([f], cache_root=tmp_path) + assert _cid(tmp_path, tmp_path / "app/javascript" / "react") not in _targets(r) + + +def test_relative_import_unaffected_by_baseurl(tmp_path): + _write(tmp_path / "jsconfig.json", + '{\n "compilerOptions": { "baseUrl": "app/javascript" }\n}\n') + local = _write(tmp_path / "app/javascript" / "packs" / "Local.js", + "export default 1;\n") + f = _write(tmp_path / "app/javascript" / "packs" / "main.js", + "import L from './Local.js';\nexport default L;\n") + r = extract([f], cache_root=tmp_path) + assert _cid(tmp_path, local) in _targets(r) + + +def test_no_baseurl_declared_changes_nothing(tmp_path): + # Without baseUrl there is no fallback root, so a bare specifier stays + # external and must not resolve to a file under the config dir. + _write(tmp_path / "jsconfig.json", '{\n "compilerOptions": {}\n}\n') + _write(tmp_path / "mods" / "Widget.js", "export default 1;\n") + f = _write(tmp_path / "packs" / "d.js", + "import W from 'mods/Widget.js';\nexport default W;\n") + r = extract([f], cache_root=tmp_path) + assert _cid(tmp_path, tmp_path / "mods" / "Widget.js") not in _targets(r) + + +def test_tsconfig_wins_when_both_configs_present(tmp_path): + # tsconfig.json is the authoritative config when both exist (tsc/VS Code + # only fall back to jsconfig.json when there is no tsconfig.json). + _write(tmp_path / "tsconfig.json", + '{\n "compilerOptions": { "baseUrl": "ts_root" }\n}\n') + _write(tmp_path / "jsconfig.json", + '{\n "compilerOptions": { "baseUrl": "js_root" }\n}\n') + ts_hit = _write(tmp_path / "ts_root" / "mods" / "W.js", "export default 1;\n") + _write(tmp_path / "js_root" / "mods" / "W.js", "export default 2;\n") + f = _write(tmp_path / "ts_root" / "packs" / "d.js", + "import W from 'mods/W.js';\nexport default W;\n") + r = extract([f], cache_root=tmp_path) + targets = _targets(r) + assert _cid(tmp_path, ts_hit) in targets + assert _cid(tmp_path, tmp_path / "js_root" / "mods" / "W.js") not in targets diff --git a/tests/test_labeling.py b/tests/test_labeling.py index d3a68067c..bb7bc3c9d 100644 --- a/tests/test_labeling.py +++ b/tests/test_labeling.py @@ -82,7 +82,7 @@ def test_label_cli_passes_model_override(tmp_path, monkeypatch): captured = {} def fake_generate(G, communities, *, backend=None, model=None, gods=None, - quiet=False, max_concurrency=4, batch_size=100): + quiet=False, max_concurrency=4, batch_size=100, usage_out=None): captured["backend"] = backend captured["model"] = model captured["max_concurrency"] = max_concurrency @@ -143,7 +143,7 @@ def test_label_cli_missing_only_preserves_existing_labels(tmp_path, monkeypatch) captured = {} def fake_generate(G, communities, *, backend=None, model=None, gods=None, - quiet=False, max_concurrency=4, batch_size=100): + quiet=False, max_concurrency=4, batch_size=100, usage_out=None): captured["communities"] = dict(communities) return {1: "Payment Flow"}, "llm" @@ -410,3 +410,143 @@ def test_label_communities_forces_serial_for_ollama(monkeypatch): monkeypatch.delenv("GRAPHIFY_OLLAMA_PARALLEL", raising=False) label_communities(G, communities, backend="ollama", batch_size=1, max_concurrency=8) assert state["peak"] == 1, "ollama must be forced serial" + + +def test_label_communities_salvages_truncated_reply(monkeypatch): + # #1690: a reply truncated mid-object (a stingy token budget or model + # preamble) used to hard-fail the whole batch with `Expecting value: line 1 + # column 6`. The complete pairs that arrived are now salvaged. + G, communities = _graph() + monkeypatch.setattr( + "graphify.llm._call_llm", + lambda p, *, backend, max_tokens=200: '{"0": "Order Management", "1":', + ) + labels = label_communities(G, communities, backend="gemini") + assert labels[0] == "Order Management" # salvaged + assert labels[1] == "Community 1" # truncated cid falls back to placeholder + + +def test_label_communities_accumulates_token_usage(monkeypatch): + # #1694: cluster-only mode reported zero labeling cost because token usage + # from the naming LLM calls was never accumulated. label_communities now + # fills a caller-supplied usage_out accumulator, summed across all batches. + G, communities = _many_communities(6) + + def fake_call(prompt, *, backend, max_tokens=200, usage_out=None): + if usage_out is not None: + usage_out["input"] = usage_out.get("input", 0) + 100 + usage_out["output"] = usage_out.get("output", 0) + 10 + # one name per community id present in this batch + cids = [int(line.split()[1].rstrip(":")) for line in prompt.splitlines() + if line.startswith("Community ")] + return json.dumps({str(c): f"Name {c}" for c in cids}) + + monkeypatch.setattr("graphify.llm._call_llm", fake_call) + usage = {"input": 0, "output": 0} + # batch_size=2 -> 3 batches, run serially so the count is deterministic + labels = label_communities( + G, communities, backend="gemini", batch_size=2, max_concurrency=1, + usage_out=usage, + ) + assert len(labels) == 6 + assert usage == {"input": 300, "output": 30} # 3 batches * (100, 10) + + +def test_label_communities_counts_tokens_for_failed_batch(monkeypatch): + # A batch whose reply can't be parsed was still billed by the provider, so + # its tokens must be counted even though it contributes no label (#1694). + G, communities = _graph() + + def fake_call(prompt, *, backend, max_tokens=200, usage_out=None): + if usage_out is not None: + usage_out["input"] = usage_out.get("input", 0) + 50 + usage_out["output"] = usage_out.get("output", 0) + 5 + return "not json at all" + + monkeypatch.setattr("graphify.llm._call_llm", fake_call) + usage = {"input": 0, "output": 0} + # single community -> no split retry; the only batch fails to parse, so + # label_communities re-raises (every batch failed) after counting tokens. + G2 = nx.Graph() + G2.add_node("a", label="alpha") + with pytest.raises((ValueError, json.JSONDecodeError)): + label_communities( + G2, {0: ["a"]}, backend="gemini", usage_out=usage, + ) + assert usage == {"input": 50, "output": 5} + + +def _two_community_graph(out): + """Two disconnected components -> two stable communities, each hub-labelled + by its own node.""" + graph = { + "directed": False, "multigraph": False, + "nodes": [ + {"id": "orders", "label": "OrderService", "community": 0}, + {"id": "order_db", "label": "OrderDB", "community": 0}, + {"id": "payments", "label": "PaymentService", "community": 1}, + {"id": "pay_db", "label": "PayDB", "community": 1}, + ], + "links": [ + {"source": "orders", "target": "order_db", "relation": "calls"}, + {"source": "payments", "target": "pay_db", "relation": "calls"}, + ], + } + (out / "graph.json").write_text(json.dumps(graph), encoding="utf-8") + + +def test_cluster_only_no_label_does_not_persist_placeholders(tmp_path, monkeypatch): + """#2073: --no-label must not write .graphify_labels.json with 'Community N' + placeholders (which the reuse path would then treat as fresh forever). A + later normal run must produce real (non-placeholder) labels.""" + import graphify.__main__ as cli + out = tmp_path / "graphify-out" + out.mkdir() + _two_community_graph(out) + labels_path = out / ".graphify_labels.json" + + monkeypatch.setattr(cli, "_check_skill_version", lambda _: None) + monkeypatch.setattr("graphify.export.to_html", lambda *a, **k: None) + # No-backend fallback shape: returns placeholders, which must not clobber hubs. + monkeypatch.setattr("graphify.llm.generate_community_labels", + lambda G, comms, **k: ({cid: f"Community {cid}" for cid in comms}, "none")) + + monkeypatch.setattr(sys, "argv", ["graphify", "cluster-only", str(tmp_path), "--no-label", "--no-viz"]) + cli.main() + assert not labels_path.exists(), "--no-label persisted a placeholder labels file (#2073)" + assert not (out / ".graphify_labels.json.sig").exists() + + # A later normal run generates real labels (no sticky placeholders blocking it). + monkeypatch.setattr(sys, "argv", ["graphify", "cluster-only", str(tmp_path), "--no-viz"]) + cli.main() + assert labels_path.exists() + saved = json.loads(labels_path.read_text(encoding="utf-8")) + assert saved, "no labels written on the normal run" + assert not any(v == f"Community {k}" for k, v in saved.items()), ( + f"real labels expected, got placeholders: {saved} (#2073)" + ) + + +def test_cluster_only_heals_persisted_placeholder_but_reuses_genuine(tmp_path, monkeypatch): + """#2073: an already-polluted sidecar (a placeholder for one community, a + genuine label for another) self-heals — the placeholder is replaced by the + hub name while the genuine label is reused, with no LLM call.""" + import graphify.__main__ as cli + out = tmp_path / "graphify-out" + out.mkdir() + _two_community_graph(out) + labels_path = out / ".graphify_labels.json" + # Polluted state: community 0 is a stuck placeholder, community 1 is genuine. + labels_path.write_text(json.dumps({"0": "Community 0", "1": "Payment Flow"}), encoding="utf-8") + + monkeypatch.setattr(cli, "_check_skill_version", lambda _: None) + monkeypatch.setattr("graphify.export.to_html", lambda *a, **k: None) + def _fail_generate(*a, **k): + raise AssertionError("generate_community_labels must not be called on the reuse path") + monkeypatch.setattr("graphify.llm.generate_community_labels", _fail_generate) + + monkeypatch.setattr(sys, "argv", ["graphify", "cluster-only", str(tmp_path), "--no-viz"]) + cli.main() + saved = json.loads(labels_path.read_text(encoding="utf-8")) + assert saved["0"] != "Community 0", "stuck placeholder was not healed (#2073)" + assert saved["1"] == "Payment Flow", "genuine label was not reused" diff --git a/tests/test_languages.py b/tests/test_languages.py index 9aa64b526..aff689d2a 100644 --- a/tests/test_languages.py +++ b/tests/test_languages.py @@ -115,6 +115,15 @@ def test_java_no_dangling_edges(): assert e["source"] in node_ids +def test_java_enum_constants_have_case_of_edge(): + r = extract_java(FIXTURES / "sample.java") + labels = _labels(r) + assert "OK" in labels + assert "GAME_DONE" in labels + assert ("ErrorCode", "OK") in _edge_labels(r, "case_of") + assert ("ErrorCode", "GAME_DONE") in _edge_labels(r, "case_of") + + # ── C ──────────────────────────────────────────────────────────────────────── def test_c_no_error(): @@ -544,6 +553,39 @@ def test_java_enum_and_annotation_declarations_are_type_nodes(tmp_path): assert definitions["Audited"].get("source_file") == str(source) +def test_nested_types_contained_by_enclosing_type(tmp_path): + """#2040: a nested class/object/trait's `contains` edge sources from its + ENCLOSING type, not the file node; top-level types still source from the file + (keeping the tree connected: file -> Outer -> Inner).""" + # Java inner class + j = tmp_path / "Outer.java" + j.write_text("class Outer {\n class Inner { void m() {} }\n}\n") + cj = _edge_labels(extract_java(j), "contains") + assert ("Outer.java", "Outer") in cj # top-level still file-sourced + assert ("Outer", "Inner") in cj # nested sourced from enclosing type + assert ("Outer.java", "Inner") not in cj # NOT from the file (the bug) + + # Scala nested class + object (the issue's repro shape) + s = tmp_path / "Outer.scala" + s.write_text("class Outer {\n class Inner\n object Obj\n}\n") + cs = _edge_labels(extract_scala(s), "contains") + assert ("Outer.scala", "Outer") in cs + assert ("Outer", "Inner") in cs + assert ("Outer", "Obj") in cs + assert ("Outer.scala", "Inner") not in cs + + +def test_csharp_nested_type_gets_containment_edge(tmp_path): + """#2040 for C#: the nested type now gets a real `contains` edge from its + enclosing type (the is_nested_type flag it already carried is retained and + covered by test_csharp_type_resolution).""" + c = tmp_path / "N.cs" + c.write_text("namespace N {\n class Outer {\n class Inner {}\n }\n}\n") + cc = _edge_labels(extract_csharp(c), "contains") + assert ("Outer", "Inner") in cc + assert ("N.cs", "Inner") not in cc + + def test_csharp_field_type_references_have_field_context(): r = extract_csharp(FIXTURES / "sample.cs") refs = _references(r) @@ -606,6 +648,14 @@ def test_kotlin_finds_function(): r = extract_kotlin(FIXTURES / "sample.kt") assert any("createClient" in l for l in _labels(r)) +def test_kotlin_enum_entries_have_case_of_edge(): + # #1700 (Kotlin half): enum entries must be nodes with case_of edges to the enum. + r = extract_kotlin(FIXTURES / "sample.kt") + labels = _labels(r) + assert "NORMAL" in labels and "GROUP" in labels and "SYSTEM" in labels + assert ("ChatType", "NORMAL") in _edge_labels(r, "case_of") + assert ("ChatType", "SYSTEM") in _edge_labels(r, "case_of") + def test_kotlin_emits_in_file_calls(): """Regression test for the call-walker `simple_identifier` / `identifier` rename — see graphify-kmp's PythonParityTest.""" @@ -639,6 +689,27 @@ def test_kotlin_parameter_return_generic_and_field_contexts(): assert ("run", "DataProcessor") in _edge_labels(r, "references", "generic_arg") assert ("DataProcessor", "Result") in _edge_labels(r, "references", "field") +def test_kotlin_builtin_types_not_emitted_as_references(): + # kotlin.* scalar/collection/core types used as parameter, return, or field + # types carry no useful graph meaning: they never resolve to a project node, + # so emitting `references` edges to them is pure noise (mirrors the Java + # _JAVA_BUILTIN_TYPES / Python _PYTHON_ANNOTATION_NOISE handling). + r = extract_kotlin(FIXTURES / "sample.kt") + ref_targets = {target for (_, target) in _edge_labels(r, "references")} + for builtin in ("String", "Int"): + assert builtin not in ref_targets, ( + f"builtin type {builtin!r} should not be a references target" + ) + +def test_kotlin_user_types_still_emit_references(): + # Guard against over-filtering: a user-defined class sharing its name with a + # common domain-modeling identifier (Result) must still resolve to a real + # edge - the builtin filter is a fixed name list, so it must stay narrow + # enough not to swallow common user-chosen names like a sealed-class "Result". + r = extract_kotlin(FIXTURES / "sample.kt") + assert ("DataProcessor", "Result") in _edge_labels(r, "references", "field") + assert ("run", "DataProcessor") in _edge_labels(r, "references", "parameter_type") + # ── Scala ───────────────────────────────────────────────────────────────────── diff --git a/tests/test_llm_backends.py b/tests/test_llm_backends.py index 79f64027e..c1392ded5 100644 --- a/tests/test_llm_backends.py +++ b/tests/test_llm_backends.py @@ -22,6 +22,34 @@ def _clear_backend_env(monkeypatch): monkeypatch.delenv(env_key, raising=False) +def test_resolve_ollama_base_url_prefers_base_url(monkeypatch): + monkeypatch.setenv("OLLAMA_BASE_URL", "custom-base-url") + monkeypatch.setenv("OLLAMA_HOST", "ignored-host:11434") + + assert llm._resolve_ollama_base_url("default-url") == "custom-base-url" + + +def test_resolve_ollama_base_url_normalizes_host_without_scheme(monkeypatch): + monkeypatch.delenv("OLLAMA_BASE_URL", raising=False) + monkeypatch.setenv("OLLAMA_HOST", "myhost:11434") + + assert llm._resolve_ollama_base_url("default-url") == "http://myhost:11434/v1" + + +def test_resolve_ollama_base_url_preserves_normalized_host(monkeypatch): + monkeypatch.delenv("OLLAMA_BASE_URL", raising=False) + monkeypatch.setenv("OLLAMA_HOST", "https://myhost:11434/v1") + + assert llm._resolve_ollama_base_url("default-url") == "https://myhost:11434/v1" + + +def test_resolve_ollama_base_url_returns_default_without_env(monkeypatch): + monkeypatch.delenv("OLLAMA_BASE_URL", raising=False) + monkeypatch.delenv("OLLAMA_HOST", raising=False) + + assert llm._resolve_ollama_base_url("default-url") == "default-url" + + def test_gemini_accepts_gemini_api_key(monkeypatch): _clear_backend_env(monkeypatch) monkeypatch.setenv("GEMINI_API_KEY", "gemini-key") @@ -578,6 +606,53 @@ def test_call_openai_compat_extra_body_wins_over_moonshot_default(monkeypatch): assert captured["extra_body"] == {"thinking": {"type": "enabled"}} +# --------------------------------------------------------------------------- +# GRAPHIFY_DISABLE_THINKING: opt-in disable-thinking for reasoning models like +# deepseek-v4-flash. Off by default — disabling thinking trades a rare reasoning +# leak for lower extraction quality/coverage, so it must not be forced (#1621). +# --------------------------------------------------------------------------- + + +def test_deepseek_thinking_on_by_default(monkeypatch): + monkeypatch.delenv("GRAPHIFY_DISABLE_THINKING", raising=False) + captured = _install_capturing_openai(monkeypatch) + + llm._call_openai_compat( + "https://api.deepseek.com", "sk", "deepseek-v4-flash", + "u", temperature=0, max_completion_tokens=8192, backend="deepseek", + ) + + eb = captured.get("extra_body") + assert eb is None or "thinking" not in eb, "thinking must NOT be disabled by default" + + +def test_deepseek_thinking_disabled_via_env(monkeypatch): + monkeypatch.setenv("GRAPHIFY_DISABLE_THINKING", "1") + captured = _install_capturing_openai(monkeypatch) + + llm._call_openai_compat( + "https://api.deepseek.com", "sk", "deepseek-v4-flash", + "u", temperature=0, max_completion_tokens=8192, backend="deepseek", + ) + + assert captured["extra_body"] == {"thinking": {"type": "disabled"}} + + +def test_explicit_extra_body_wins_over_thinking_env(monkeypatch): + # A provider-supplied extra_body is an explicit request-shape choice and must + # take precedence over the env toggle. + monkeypatch.setenv("GRAPHIFY_DISABLE_THINKING", "1") + captured = _install_capturing_openai(monkeypatch) + + llm._call_openai_compat( + "https://api.deepseek.com", "sk", "deepseek-v4-flash", + "u", temperature=0, max_completion_tokens=8192, backend="deepseek", + extra_body={"thinking": {"type": "enabled"}}, + ) + + assert captured["extra_body"] == {"thinking": {"type": "enabled"}} + + def test_call_openai_compat_explicit_extra_body_skips_ollama_auto_derive(monkeypatch): # An explicit extra_body means "I own this request shape" — Ollama's # num_ctx auto-derive (a default) must step aside or we'd clobber it. diff --git a/tests/test_merge_chunks_validation.py b/tests/test_merge_chunks_validation.py new file mode 100644 index 000000000..b2d532591 --- /dev/null +++ b/tests/test_merge_chunks_validation.py @@ -0,0 +1,150 @@ +"""Tests that `graphify merge-chunks` validates untrusted subagent chunk JSON. + +merge-chunks concatenates agent-written `.graphify_chunk_*.json` files. Those are +untrusted output, so each is run through `validate_semantic_fragment` (caps + the +node/edge ID charset that blocks path-escape). An invalid chunk is skipped with a +warning; valid chunks still merge, but an all-invalid input set fails closed. +""" +import json + +import graphify.__main__ as mainmod +import pytest + + +def _write(path, obj): + path.write_text(json.dumps(obj), encoding="utf-8") + + +def _run_merge(monkeypatch, argv): + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr(mainmod.sys, "argv", argv) + mainmod.main() + + +def test_merge_chunks_skips_chunk_with_path_escape_id(tmp_path, monkeypatch, capsys): + good = tmp_path / ".graphify_chunk_0.json" + _write(good, {"nodes": [{"id": "pkg.mod.good", "label": "G"}], "edges": [], "hyperedges": []}) + bad = tmp_path / ".graphify_chunk_1.json" + # A node id with a path separator would escape the chunk directory (#825). + _write(bad, {"nodes": [{"id": "../../etc/passwd", "label": "B"}], "edges": [], "hyperedges": []}) + out = tmp_path / "merged.json" + + _run_merge(monkeypatch, ["graphify", "merge-chunks", str(good), str(bad), "--out", str(out)]) + + merged = json.loads(out.read_text()) + assert {n["id"] for n in merged["nodes"]} == {"pkg.mod.good"} + captured = capsys.readouterr() + assert "skipping invalid chunk" in captured.err + assert "Merged 1 of 2 chunks" in captured.out + + +def test_merge_chunks_fails_closed_when_every_chunk_is_invalid(tmp_path, monkeypatch, capsys): + bad = tmp_path / ".graphify_chunk_0.json" + _write(bad, {"nodes": "not-a-list", "edges": []}) + out = tmp_path / "merged.json" + out.write_text('{"previous": "semantic result"}', encoding="utf-8") + + with pytest.raises(SystemExit) as exc: + _run_merge(monkeypatch, ["graphify", "merge-chunks", str(bad), "--out", str(out)]) + + assert exc.value.code == 1 + assert json.loads(out.read_text()) == {"previous": "semantic result"} + err = capsys.readouterr().err + assert "skipping invalid chunk" in err + assert "no valid chunks to merge" in err + + +def test_merge_chunks_accepts_valid_empty_chunk(tmp_path, monkeypatch): + """A valid fragment may legitimately contain no entities; it still counts.""" + empty = tmp_path / ".graphify_chunk_0.json" + _write(empty, {"nodes": [], "edges": [], "hyperedges": []}) + out = tmp_path / "merged.json" + + _run_merge(monkeypatch, ["graphify", "merge-chunks", str(empty), "--out", str(out)]) + + merged = json.loads(out.read_text()) + assert merged["nodes"] == [] + assert merged["edges"] == [] + + +def test_merge_chunks_fails_closed_without_chunk_arguments(tmp_path, monkeypatch, capsys): + out = tmp_path / "merged.json" + + with pytest.raises(SystemExit) as exc: + _run_merge(monkeypatch, ["graphify", "merge-chunks", "--out", str(out)]) + + assert exc.value.code == 1 + assert not out.exists() + assert "no valid chunks to merge" in capsys.readouterr().err + + +def test_merge_chunks_fails_closed_on_unmatched_glob(tmp_path, monkeypatch, capsys): + out = tmp_path / "merged.json" + out.write_text('{"previous": true}', encoding="utf-8") + unmatched = str(tmp_path / ".graphify_chunk_*.json") + + with pytest.raises(SystemExit) as exc: + _run_merge(monkeypatch, ["graphify", "merge-chunks", unmatched, "--out", str(out)]) + + assert exc.value.code == 1 + assert json.loads(out.read_text()) == {"previous": True} + err = capsys.readouterr().err + assert "skipping invalid chunk" in err + assert "no valid chunks to merge" in err + + +def test_merge_chunks_accepts_synonym_file_type(tmp_path, monkeypatch): + # file_type synonyms (markdown/tool/framework/...) are coerced by build, not + # a validation failure — the chunk must merge, not be silently dropped (#840). + c = tmp_path / ".graphify_chunk_0.json" + _write(c, {"nodes": [{"id": "pkg.readme", "label": "Readme", "file_type": "markdown"}, + {"id": "pkg.tool", "label": "Tool", "file_type": "tool"}], + "edges": [], "hyperedges": []}) + out = tmp_path / "merged.json" + _run_merge(monkeypatch, ["graphify", "merge-chunks", str(c), "--out", str(out)]) + merged = json.loads(out.read_text()) + assert {n["id"] for n in merged["nodes"]} == {"pkg.readme", "pkg.tool"} + + +def test_merge_chunks_accepts_unicode_id(tmp_path, monkeypatch): + # build's normalize_id preserves Unicode identifiers; validation must not + # reject a chunk that uses them. + c = tmp_path / ".graphify_chunk_0.json" + _write(c, {"nodes": [{"id": "mod_处理数据", "label": "handler", "file_type": "code"}], + "edges": [], "hyperedges": []}) + out = tmp_path / "merged.json" + _run_merge(monkeypatch, ["graphify", "merge-chunks", str(c), "--out", str(out)]) + merged = json.loads(out.read_text()) + assert {n["id"] for n in merged["nodes"]} == {"mod_处理数据"} + + +def test_validate_semantic_fragment_accepts_synonyms_and_unicode(): + from graphify.semantic_cleanup import validate_semantic_fragment + frag = {"nodes": [{"id": "mod_处理", "file_type": "markdown"}, + {"id": "a.b::C.d", "file_type": "tool"}], + "edges": [], "hyperedges": []} + assert validate_semantic_fragment(frag) == [] + + +def test_validate_semantic_fragment_still_blocks_path_escape(): + from graphify.semantic_cleanup import validate_semantic_fragment + errs = validate_semantic_fragment({"nodes": [{"id": "../../etc/passwd"}], + "edges": [], "hyperedges": []}) + assert errs + + +def test_merge_chunks_merges_valid_chunks(tmp_path, monkeypatch): + c0 = tmp_path / ".graphify_chunk_0.json" + _write(c0, {"nodes": [{"id": "a", "label": "A"}], "edges": [], "hyperedges": [], + "input_tokens": 10, "output_tokens": 5}) + c1 = tmp_path / ".graphify_chunk_1.json" + _write(c1, {"nodes": [{"id": "b", "label": "B"}], "edges": [], "hyperedges": [], + "input_tokens": 7, "output_tokens": 3}) + out = tmp_path / "merged.json" + + _run_merge(monkeypatch, ["graphify", "merge-chunks", str(c0), str(c1), "--out", str(out)]) + + merged = json.loads(out.read_text()) + assert {n["id"] for n in merged["nodes"]} == {"a", "b"} + assert merged["input_tokens"] == 17 + assert merged["output_tokens"] == 8 diff --git a/tests/test_merge_graphs_cli.py b/tests/test_merge_graphs_cli.py index f3e0e05d2..5ff06b391 100644 --- a/tests/test_merge_graphs_cli.py +++ b/tests/test_merge_graphs_cli.py @@ -46,3 +46,49 @@ def test_merge_graphs_mixed_directed_and_multigraph(tmp_path): assert {"r1::x", "r2::y", "r3::z"} <= ids or len(ids) == 3 assert data.get("directed") is False assert data.get("multigraph") is False + + +def test_merge_graphs_same_named_repo_dirs_do_not_collapse(tmp_path): + # #1729: two graphs under a same-named repo dir (src/graphify-out and + # frontend/src/graphify-out both → tag "src") share the `src::` prefix, so a + # bare `app` node from each collapsed into one — silently merging unrelated + # entities and inventing cross-runtime edges. Distinct tags must keep them apart. + a = tmp_path / "src" / "graphify-out" / "graph.json" + b = tmp_path / "frontend" / "src" / "graphify-out" / "graph.json" + a.parent.mkdir(parents=True, exist_ok=True) + b.parent.mkdir(parents=True, exist_ok=True) + a.write_text(json.dumps({"directed": False, "multigraph": False, "nodes": [ + {"id": "app", "label": "app.js", "source_file": "app.js"}], "links": []})) + b.write_text(json.dumps({"directed": False, "multigraph": False, "nodes": [ + {"id": "app", "label": "App.jsx", "source_file": "App.jsx"}], "links": []})) + out = tmp_path / "merged.json" + + r = _run(["merge-graphs", str(a), str(b), "--out", str(out)], tmp_path) + assert r.returncode == 0, r.stderr + data = json.loads(out.read_text()) + app_nodes = [n for n in data["nodes"] if n["id"].endswith("::app")] + assert len(app_nodes) == 2, f"both app nodes must survive; got {[n['id'] for n in app_nodes]}" + labels = {n.get("label") for n in app_nodes} + assert labels == {"app.js", "App.jsx"}, f"both entities preserved; got {labels}" + + +def test_distinct_repo_tags_unit(tmp_path): + from graphify.build import distinct_repo_tags + # distinct repo dirs pass through unchanged + assert distinct_repo_tags([ + Path("backend/graphify-out/graph.json"), + Path("web/graphify-out/graph.json"), + ]) == ["backend", "web"] + # same-named repo dirs are widened to stay distinct + tags = distinct_repo_tags([ + Path("proj/src/graphify-out/graph.json"), + Path("proj/frontend/src/graphify-out/graph.json"), + ]) + assert len(set(tags)) == 2, tags + # a repeated dir name triple still yields all-distinct tags (index fallback) + tags3 = distinct_repo_tags([ + Path("a/src/graphify-out/graph.json"), + Path("b/src/graphify-out/graph.json"), + Path("c/src/graphify-out/graph.json"), + ]) + assert len(set(tags3)) == 3, tags3 diff --git a/tests/test_multilang.py b/tests/test_multilang.py index 5ad78f724..4bac41bc4 100644 --- a/tests/test_multilang.py +++ b/tests/test_multilang.py @@ -513,3 +513,76 @@ def test_sql_schema_qualified_alter_fk(): for e in fk_edges: assert e["source"] in node_ids, f"dangling source: {e['source']}" assert e["target"] in node_ids, f"dangling target: {e['target']}" + +def test_sql_plpgsql_functions_survive_parse_errors(): + """PL/pgSQL bodies make tree-sitter-sql emit ERROR nodes; the functions + must still be extracted (#1910), without cascading into later statements.""" + r = _extract_sql_or_skip("sample_plpgsql.sql") + labels = [n["label"] for n in r["nodes"]] + # Both PL/pgSQL functions extracted, schema-qualified name kept whole + assert "exposed.important_function()" in labels + assert "tagged_quote_fn()" in labels + # Tables before and after the broken functions still extract + assert any("accounts" in l for l in labels) + assert any("audit_log" in l for l in labels) + # No spurious or empty nodes from the error recovery + for l in labels: + assert l, "empty node label" + assert l != "ERROR" + # Every function got a contains edge from the file node + contains_targets = {e["target"] for e in r["edges"] if e["relation"] == "contains"} + fn_ids = {n["id"] for n in r["nodes"] if n["label"].endswith("()")} + assert fn_ids <= contains_targets + +def test_sql_plpgsql_clean_function_not_double_emitted(): + """A cleanly-parsed LANGUAGE sql function in the same file is emitted once.""" + r = _extract_sql_or_skip("sample_plpgsql.sql") + labels = [n["label"] for n in r["nodes"]] + assert labels.count("plain_sql_fn()") == 1 + # And nothing else is duplicated either + ids = [n["id"] for n in r["nodes"]] + assert len(ids) == len(set(ids)) + +def test_sql_quoted_plpgsql_routines_are_recovered(): + """#2180: quoted identifiers must not defeat the ERROR-node name recovery. + + Generated PostgreSQL DDL (Supabase-style dumps, for example) quotes every + identifier: CREATE OR REPLACE FUNCTION "public"."fn"(...). The recovery + pattern matched a bare [\\w$.]+, which stops at the leading quote, so every + routine whose body also failed to parse -- RAISE, PERFORM, :=, IF..THEN, + bare NULL; -- was dropped with no warning and exit code 0. The same body + with an *unquoted* name recovered fine, which is why the drop looked like it + depended only on the body statement. + """ + r = _extract_sql_or_skip("sample_plpgsql_quoted.sql") + labels = [n["label"] for n in r["nodes"]] + for name in ( + "raise_exception_fn", + "raise_notice_fn", + "perform_fn", + "assign_fn", + "if_then_fn", + "null_body_fn", + "quoted_proc", + ): + assert f'"public"."{name}"()' in labels, f"{name} dropped from a quoted-DDL file (#2180)" + +def test_sql_quoted_plpgsql_file_stays_clean(): + """The #2180 recovery must not add junk, duplicates, or drop the tables.""" + r = _extract_sql_or_skip("sample_plpgsql_quoted.sql") + labels = [n["label"] for n in r["nodes"]] + # Tables before and after the unparseable routines still extract. + assert any("accounts" in l for l in labels) + assert any("audit_log" in l for l in labels) + # No empty or ERROR labels leaked out of the recovery. + for l in labels: + assert l, "empty node label" + assert l != "ERROR" + # Nothing emitted twice (a routine must not come from both paths). + ids = [n["id"] for n in r["nodes"]] + assert len(ids) == len(set(ids)), "duplicate node ids" + assert len(labels) == len(set(labels)), f"duplicate labels: {labels}" + # Every recovered routine is reachable from the file node. + contains_targets = {e["target"] for e in r["edges"] if e["relation"] == "contains"} + fn_ids = {n["id"] for n in r["nodes"] if n["label"].endswith("()")} + assert fn_ids <= contains_targets diff --git a/tests/test_ollama_retry_cap.py b/tests/test_ollama_retry_cap.py new file mode 100644 index 000000000..5ad43cc73 --- /dev/null +++ b/tests/test_ollama_retry_cap.py @@ -0,0 +1,60 @@ +"""#1686 - a wedged local Ollama request must not multiply --api-timeout by the +SDK's 6 transient-error retries into a ~20min block. Ollama defaults to 0 SDK +retries so the timeout is the effective wall-clock bound; an explicit +GRAPHIFY_MAX_RETRIES still wins. +""" +from __future__ import annotations + +from unittest.mock import MagicMock + +import graphify.llm as llm + + +def _capture_client_kwargs(monkeypatch): + captured: dict = {} + + def _factory(**kwargs): + captured.update(kwargs) + client = MagicMock() + resp = MagicMock() + resp.choices[0].message.content = '{"nodes": [], "edges": [], "hyperedges": []}' + resp.choices[0].finish_reason = "stop" + resp.usage.prompt_tokens = 1 + resp.usage.completion_tokens = 1 + client.chat.completions.create.return_value = resp + return client + + monkeypatch.setattr("openai.OpenAI", _factory) + return captured + + +def test_ollama_defaults_to_zero_sdk_retries(monkeypatch): + monkeypatch.delenv("GRAPHIFY_MAX_RETRIES", raising=False) + captured = _capture_client_kwargs(monkeypatch) + llm._call_openai_compat("http://localhost:11434/v1", "ollama", "m", + "def f(): pass", backend="ollama") + assert captured.get("max_retries") == 0 + + +def test_ollama_honors_explicit_max_retries(monkeypatch): + monkeypatch.setenv("GRAPHIFY_MAX_RETRIES", "3") + captured = _capture_client_kwargs(monkeypatch) + llm._call_openai_compat("http://localhost:11434/v1", "ollama", "m", + "def f(): pass", backend="ollama") + assert captured.get("max_retries") == 3 + + +def test_cloud_backend_keeps_default_retries(monkeypatch): + monkeypatch.delenv("GRAPHIFY_MAX_RETRIES", raising=False) + captured = _capture_client_kwargs(monkeypatch) + llm._call_openai_compat("https://api.moonshot.cn/v1", "sk-x", "m", + "def f(): pass", backend="kimi") + assert captured.get("max_retries") == 6 # default retained for rate-limited clouds + + +def test_api_timeout_is_passed_to_client(monkeypatch): + monkeypatch.setenv("GRAPHIFY_API_TIMEOUT", "180") + captured = _capture_client_kwargs(monkeypatch) + llm._call_openai_compat("http://localhost:11434/v1", "ollama", "m", + "def f(): pass", backend="ollama") + assert captured.get("timeout") == 180.0 diff --git a/tests/test_partial_cache.py b/tests/test_partial_cache.py new file mode 100644 index 000000000..66f4f7659 --- /dev/null +++ b/tests/test_partial_cache.py @@ -0,0 +1,187 @@ +"""Tests for partial-extraction cache promotion. + +A truncated LLM chunk (`finish_reason="length"` that could not be recovered by +splitting, or a max-depth adaptive-retry give-up) yields an incomplete node set. +It is tagged with an internal ``_partial`` marker; ``save_semantic_cache`` stamps +that file's entry ``partial: True``, and ``load_cached`` then treats a partial +entry as a cache MISS so the file is re-dispatched instead of served forever. +""" + +from graphify import llm +from graphify.cache import ( + save_semantic_cache, + load_cached, + _group_has_partial_marker, +) + + +def _doc(tmp_path): + doc = tmp_path / "doc.md" + doc.write_text("# Heading\nsome prose\n", encoding="utf-8") + return doc + + +def test_intrinsic_partial_marker_makes_entry_a_cache_miss(tmp_path): + doc = _doc(tmp_path) + nodes = [{"id": "n1", "label": "Heading", "source_file": "doc.md", "_partial": True}] + saved = save_semantic_cache(nodes, [], root=tmp_path, prompt="P") + assert saved == 1 + # The stamped entry is present on disk, but load_cached rejects it. + assert load_cached(doc, root=tmp_path, kind="semantic", prompt="P") is None + + +def test_partial_source_files_arg_stamps_entry(tmp_path): + doc = _doc(tmp_path) + # No intrinsic marker; partial-ness comes only from the explicit arg. + nodes = [{"id": "n1", "label": "Heading", "source_file": "doc.md"}] + save_semantic_cache(nodes, [], root=tmp_path, prompt="P", partial_source_files=["doc.md"]) + assert load_cached(doc, root=tmp_path, kind="semantic", prompt="P") is None + + +def test_non_partial_entry_loads_normally(tmp_path): + doc = _doc(tmp_path) + nodes = [{"id": "n1", "label": "Heading", "source_file": "doc.md"}] + save_semantic_cache(nodes, [], root=tmp_path, prompt="P") + loaded = load_cached(doc, root=tmp_path, kind="semantic", prompt="P") + assert loaded is not None + assert len(loaded["nodes"]) == 1 + + +def test_partial_entry_self_heals_on_complete_reextraction(tmp_path): + doc = _doc(tmp_path) + partial = [{"id": "n1", "source_file": "doc.md", "_partial": True}] + save_semantic_cache(partial, [], root=tmp_path, prompt="P") + assert load_cached(doc, root=tmp_path, kind="semantic", prompt="P") is None + # A later complete extraction overwrites the same content-hash key with a + # non-partial entry, which then serves normally. + complete = [ + {"id": "n1", "source_file": "doc.md"}, + {"id": "n2", "source_file": "doc.md"}, + ] + save_semantic_cache(complete, [], root=tmp_path, prompt="P") + loaded = load_cached(doc, root=tmp_path, kind="semantic", prompt="P") + assert loaded is not None + assert len(loaded["nodes"]) == 2 + + +def test_merge_existing_accumulates_slices_and_stays_partial(tmp_path): + """A file sliced across chunks: an earlier truncated slice must not be + dropped (nor the entry promoted to complete) by a later clean slice's + merge_existing checkpoint. The union keeps both slices and the entry stays + partial until a fully-clean re-extraction overwrites it.""" + doc = _doc(tmp_path) + partial = [{"id": "n1", "source_file": "doc.md", "_partial": True}] + save_semantic_cache(partial, [], root=tmp_path, prompt="P") + fresh = [{"id": "n2", "source_file": "doc.md"}] + save_semantic_cache(fresh, [], root=tmp_path, prompt="P", merge_existing=True) + # Normal read is still a miss: the file had a truncated slice, so it must be + # re-dispatched rather than served. + assert load_cached(doc, root=tmp_path, kind="semantic", prompt="P") is None + # But nothing was lost — both slices are present in the accumulated entry. + peek = load_cached(doc, root=tmp_path, kind="semantic", prompt="P", allow_partial=True) + assert peek is not None + assert {n["id"] for n in peek["nodes"]} == {"n1", "n2"} + + +def test_save_stamps_partial_file_with_no_items(tmp_path): + """#1950 empty-parse gap: a chunk that truncates to an empty parse produces + NO items, so partial-ness can only come from partial_source_files. save must + still stamp such a file partial (seeding an empty group) — even when a prior + clean slice already cached it — so it is re-dispatched instead of served.""" + doc = _doc(tmp_path) + # A clean slice cached first. + save_semantic_cache([{"id": "n1", "source_file": "doc.md"}], [], root=tmp_path, prompt="P") + assert load_cached(doc, root=tmp_path, kind="semantic", prompt="P") is not None + # Now an empty-parse truncation covering the same file: no items, only the + # named partial file. The entry must flip to a miss. + save_semantic_cache([], [], root=tmp_path, prompt="P", + merge_existing=True, partial_source_files=["doc.md"]) + assert load_cached(doc, root=tmp_path, kind="semantic", prompt="P") is None + # The earlier slice's node is not lost — it stays in the partial entry. + peek = load_cached(doc, root=tmp_path, kind="semantic", prompt="P", allow_partial=True) + assert peek is not None and {n["id"] for n in peek["nodes"]} == {"n1"} + + +def test_clean_slice_does_not_repromote_empty_parse_partial(tmp_path): + """Ordering guard: once a file is partial (from an empty-parse truncation, + so no item markers), a later clean slice merging over it must keep it partial + via the carried-forward prev flag — not silently promote it to complete.""" + doc = _doc(tmp_path) + # Empty-parse partial first (no markers, only the named file). + save_semantic_cache([], [], root=tmp_path, prompt="P", partial_source_files=["doc.md"]) + assert load_cached(doc, root=tmp_path, kind="semantic", prompt="P") is None + # A later clean slice checkpoints with merge_existing and no partial arg. + save_semantic_cache([{"id": "n2", "source_file": "doc.md"}], [], root=tmp_path, + prompt="P", merge_existing=True) + # Must still be a miss — the prior truncation is unresolved. + assert load_cached(doc, root=tmp_path, kind="semantic", prompt="P") is None + + +def test_partial_files_carries_empty_parse_truncation(): + """_partial_source_files must surface a file recorded in _partial_files even + when the result has zero items (the empty-parse case).""" + import graphify.llm as llm + result = {"nodes": [], "edges": [], "hyperedges": [], "_partial_files": ["big.md"]} + assert llm._partial_source_files(result) == ["big.md"] + # And it unions with intrinsic item markers. + result2 = {"nodes": [{"id": "a", "source_file": "x.md", "_partial": True}], + "edges": [], "hyperedges": [], "_partial_files": ["big.md"]} + assert llm._partial_source_files(result2) == ["big.md", "x.md"] + + +def test_stamped_manifest_excludes_partial_files(): + """A truncated file produced output this run but is left unstamped in the + manifest (like a failed chunk) so detect_incremental re-queues it.""" + from pathlib import Path + from graphify.cli import _stamped_manifest_files + + files_by_type = {"document": ["a.md", "b.md"], "code": ["x.py"]} + sem_result = { + "nodes": [ + {"id": "1", "source_file": "a.md"}, + {"id": "2", "source_file": "b.md"}, + ], + "edges": [], "hyperedges": [], + } + out = _stamped_manifest_files(files_by_type, sem_result, Path("."), + partial_source_files={"b.md"}) + # a.md extracted cleanly -> stamped; b.md truncated -> excluded; code kept. + assert out["document"] == ["a.md"] + assert out["code"] == ["x.py"] + + +def test_group_has_partial_marker(): + assert _group_has_partial_marker({"nodes": [{"_partial": True}]}) is True + assert _group_has_partial_marker({"edges": [{"_partial": True}]}) is True + assert _group_has_partial_marker({"nodes": [{"id": "a"}], "edges": [], "hyperedges": []}) is False + assert _group_has_partial_marker({}) is False + + +def test_mark_partial_and_partial_source_files(): + result = { + "nodes": [{"id": "a", "source_file": "x.md"}], + "edges": [{"source": "a", "target": "b", "source_file": "x.md"}], + "hyperedges": [{"id": "h", "source_file": "y.md"}], + } + llm._mark_partial(result) + assert result["nodes"][0]["_partial"] is True + assert result["edges"][0]["_partial"] is True + assert result["hyperedges"][0]["_partial"] is True + assert llm._partial_source_files(result) == ["x.md", "y.md"] + + +def test_partial_source_files_empty_when_unmarked(): + result = {"nodes": [{"id": "a", "source_file": "x.md"}], "edges": [], "hyperedges": []} + assert llm._partial_source_files(result) == [] + + +def test_strip_partial_markers_removes_internal_key(): + result = { + "nodes": [{"id": "a", "_partial": True}], + "edges": [{"source": "a", "target": "b", "_partial": True}], + "hyperedges": [{"id": "h", "_partial": True}], + } + llm._strip_partial_markers(result) + assert "_partial" not in result["nodes"][0] + assert "_partial" not in result["edges"][0] + assert "_partial" not in result["hyperedges"][0] diff --git a/tests/test_pascal.py b/tests/test_pascal.py index 36c1b8747..e54564bcf 100644 --- a/tests/test_pascal.py +++ b/tests/test_pascal.py @@ -320,3 +320,26 @@ def test_dfm_dispatch_registered(): def test_dfm_detect_extension_registered(): from graphify.detect import CODE_EXTENSIONS assert ".dfm" in CODE_EXTENSIONS + + +def _dup_edges(r): + from collections import Counter + triples = Counter((e["source"], e["target"], e["relation"]) for e in r["edges"]) + return {k: v for k, v in triples.items() if v > 1} + + +def test_pascal_no_duplicate_method_edges_tree_sitter(): + """A class method appears in both the interface declaration and the + implementation; each used to emit a `method` edge to the same node, so the + graph carried doubled method/contains/inherits edges (skewing degree and + breaking the cross-file inherited-call resolver's god-node guard). Edges are + now deduped on (source, target, relation).""" + from graphify.extract import extract_pascal + r = extract_pascal(FIXTURES / "sample.pas") + assert _dup_edges(r) == {}, f"duplicate edges: {_dup_edges(r)}" + + +def test_pascal_no_duplicate_method_edges_regex(): + from graphify.extract import _extract_pascal_regex + r = _extract_pascal_regex(FIXTURES / "sample.pas") + assert _dup_edges(r) == {}, f"duplicate edges: {_dup_edges(r)}" diff --git a/tests/test_pascal_call_scoping.py b/tests/test_pascal_call_scoping.py new file mode 100644 index 000000000..3d47669a4 --- /dev/null +++ b/tests/test_pascal_call_scoping.py @@ -0,0 +1,102 @@ +"""Regression tests for scoped call resolution in the Pascal/Delphi extractor. + +Before this fix, both `extract_pascal` (tree-sitter path) and +`_extract_pascal_regex` (fallback path) resolved every call by a single +file-wide ``{method_name_lower: node_id}`` dict with no class scoping. Two +unrelated classes declaring a same-named method (a common Pascal/Delphi +pattern -- property accessors, generated wrapper classes such as TLB import +units) silently collapsed onto whichever declaration was inserted last, +producing wrong cross-class `calls` edges. See `sample_scoped_calls.pas`. +""" +from __future__ import annotations + +import pytest +from pathlib import Path + +FIXTURES = Path(__file__).parent / "fixtures" +FIXTURE_PATH = FIXTURES / "sample_scoped_calls.pas" + + +def _extractors(): + from graphify.extract import extract_pascal, _extract_pascal_regex + return [extract_pascal, _extract_pascal_regex] + + +def _class_node_id(r, class_label): + matches = [n["id"] for n in r["nodes"] if n["label"] == class_label] + assert len(matches) == 1, f"expected exactly one node labeled {class_label!r}, got {matches}" + return matches[0] + + +def _method_node_id(r, class_label, method_label): + class_id = _class_node_id(r, class_label) + node_by_id = {n["id"]: n for n in r["nodes"]} + for e in r["edges"]: + if e["relation"] == "method" and e["source"] == class_id: + node = node_by_id.get(e["target"]) + if node and node["label"] == method_label: + return node["id"] + raise AssertionError(f"no method edge {class_label}.{method_label} found") + + +def _has_call(r, src_id, tgt_id): + return any( + e["relation"] == "calls" and e["source"] == src_id and e["target"] == tgt_id + for e in r["edges"] + ) + + +@pytest.mark.parametrize("extract", [ + pytest.param(0, id="tree-sitter"), + pytest.param(1, id="regex-fallback"), +]) +def test_calls_scoped_to_own_class(extract): + r = _extractors()[extract](FIXTURE_PATH) + first_configure = _method_node_id(r, "TFirstWidget", "Configure()") + first_reset = _method_node_id(r, "TFirstWidget", "Reset()") + assert _has_call(r, first_configure, first_reset) + + +@pytest.mark.parametrize("extract", [ + pytest.param(0, id="tree-sitter"), + pytest.param(1, id="regex-fallback"), +]) +def test_calls_do_not_cross_unrelated_classes(extract): + r = _extractors()[extract](FIXTURE_PATH) + first_configure = _method_node_id(r, "TFirstWidget", "Configure()") + second_reset = _method_node_id(r, "TSecondWidget", "Reset()") + assert not _has_call(r, first_configure, second_reset), ( + "TFirstWidget.Configure must not resolve Reset() to the unrelated " + "TSecondWidget.Reset -- same-named methods on unrelated classes must " + "not collapse into a cross-class edge" + ) + + +@pytest.mark.parametrize("extract", [ + pytest.param(0, id="tree-sitter"), + pytest.param(1, id="regex-fallback"), +]) +def test_calls_scoped_other_direction(extract): + r = _extractors()[extract](FIXTURE_PATH) + second_configure = _method_node_id(r, "TSecondWidget", "Configure()") + second_reset = _method_node_id(r, "TSecondWidget", "Reset()") + first_reset = _method_node_id(r, "TFirstWidget", "Reset()") + assert _has_call(r, second_configure, second_reset) + assert not _has_call(r, second_configure, first_reset), ( + "TSecondWidget.Configure must not resolve Reset() to the unrelated " + "TFirstWidget.Reset" + ) + + +@pytest.mark.parametrize("extract", [ + pytest.param(0, id="tree-sitter"), + pytest.param(1, id="regex-fallback"), +]) +def test_calls_resolve_via_ancestor_chain(extract): + r = _extractors()[extract](FIXTURE_PATH) + derived_run = _method_node_id(r, "TDerivedWidget", "Run()") + base_prepare = _method_node_id(r, "TBaseWidget", "Prepare()") + assert _has_call(r, derived_run, base_prepare), ( + "TDerivedWidget.Run should resolve the inherited Prepare() to " + "TBaseWidget.Prepare via the inherits chain" + ) diff --git a/tests/test_pascal_resolution.py b/tests/test_pascal_resolution.py new file mode 100644 index 000000000..f48401050 --- /dev/null +++ b/tests/test_pascal_resolution.py @@ -0,0 +1,95 @@ +"""Tests for cross-file Pascal/Delphi inherited-method-call resolution. + +The per-file Pascal/Delphi extractors resolve a call to the caller's own +class, its ancestor chain, or a file-level free function -- but only within +the single file being extracted. Real Delphi/MTM-style code very commonly +splits a class across two files (a generated base class + a manual +descendant that extends it in a separate unit), so a call from the +descendant to a method it inherits from the base falls outside any one +file's own scope. graphify.pascal_resolution closes that gap as a +corpus-wide, post-extraction pass. See its module docstring for the full +rationale. + +Uses static fixtures under tests/fixtures/pascal_cross_file/ rather than +pytest's tmp_path: the Pascal extractor's cross-file class lookup +(_pascal_project_root) walks UP the directory tree looking for the highest +ancestor with 2+ .pas files, to find the project root. tmp_path lives under +the shared system temp directory, which on a dev machine can easily already +contain 2+ stray .pas files at some ancestor level (other tools' scratch +files, other tests' leftover fixtures) -- the walk-up then escalates past the +test's own directory and picks up unrelated files. tests/fixtures/ has no +such siblings above it, so it is a stable project root for these tests. +""" +from __future__ import annotations + +from pathlib import Path + +from graphify.extract import extract, extract_pascal + +FIXTURES = Path(__file__).parent / "fixtures" / "pascal_cross_file" +BASE = FIXTURES / "BaseGadget.pas" +OTHER = FIXTURES / "OtherGadget.pas" +DERIVED = FIXTURES / "DerivedGadget.pas" + + +def _find_raw_call(result: dict, callee: str) -> dict | None: + for rc in result.get("raw_calls", []): + if rc.get("callee") == callee: + return rc + return None + + +def _labels(nodes: list[dict]) -> dict[str, str]: + return {n["id"]: str(n.get("label", "")) for n in nodes} + + +def _call_edge(graph: dict, src_label: str, tgt_label: str): + labels = _labels(graph["nodes"]) + for e in graph["edges"]: + if e.get("relation") != "calls": + continue + if labels.get(e.get("source")) == src_label and labels.get(e.get("target")) == tgt_label: + return e + return None + + +def test_single_file_extraction_reports_unresolved_inherited_call(): + """Sanity check for the gap this resolver closes: the per-file extractor + alone cannot see BaseGadget.pas while extracting DerivedGadget.pas, so it + must NOT emit a `calls` edge for Run -> Prepare, and must report it via + raw_calls instead of silently dropping it.""" + r = extract_pascal(DERIVED) + assert _call_edge(r, "Run()", "Prepare()") is None + rc = _find_raw_call(r, "prepare") + assert rc is not None + assert rc["caller_nid"] + + +def test_calls_resolve_across_files_via_inherits_chain(tmp_path): + # cache_root only controls where graphify-out/cache/ is written -- it has + # no bearing on the Pascal cross-file class lookup, which is keyed off + # each source path's own project root (see module docstring). Using + # tmp_path here just keeps cache artifacts out of the repo. + graph = extract([BASE, DERIVED], cache_root=tmp_path, parallel=False) + edge = _call_edge(graph, "Run()", "Prepare()") + assert edge is not None + assert edge.get("confidence") == "EXTRACTED" + + +def test_cross_file_calls_do_not_cross_unrelated_classes(tmp_path): + """TDerivedGadget inherits only from TBaseGadget. TOtherGadget declares an + unrelated same-named Prepare in a third file -- Run() must resolve to + TBaseGadget.Prepare, never to TOtherGadget.Prepare.""" + graph = extract([BASE, OTHER, DERIVED], cache_root=tmp_path, parallel=False) + edge = _call_edge(graph, "Run()", "Prepare()") + assert edge is not None + node_by_id = {n["id"]: n for n in graph["nodes"]} + target = node_by_id[edge["target"]] + assert "BaseGadget.pas" in target.get("source_file", "") + assert "OtherGadget.pas" not in target.get("source_file", "") + + +def test_pascal_resolver_registered(): + from graphify.resolver_registry import registered_resolvers + names = {r.name for r in registered_resolvers()} + assert "pascal_inherited_calls" in names diff --git a/tests/test_path_cli.py b/tests/test_path_cli.py index de7e8837f..2584ce4b2 100644 --- a/tests/test_path_cli.py +++ b/tests/test_path_cli.py @@ -1,7 +1,12 @@ -"""Regression tests for `graphify path` arrow direction (#849).""" +"""Regression tests for `graphify path` arrow direction (#849) and determinism + +honest edge labels (#2074).""" from __future__ import annotations import json +import os +import subprocess +import sys import networkx as nx +import pytest from networkx.readwrite import json_graph import graphify.__main__ as mainmod @@ -46,3 +51,144 @@ def test_reverse_arrow(monkeypatch, tmp_path, capsys): assert "Shortest path (1 hops):" in out assert "validateSanitySession() <--calls [EXTRACTED]-- createPatchHandler()" in out assert "validateSanitySession() --calls [EXTRACTED]--> createPatchHandler()" not in out + + +def _write_misranking_graph(tmp_path): + """Graph where IDF scoring ranks a partial-token decoy above the full match. + + Query "Reject-everything judge": the decoy "Rejection Summary" prefix-matches + the rare token "reject" and out-scores "Degenerate Reject-Everything Judge" + (whose full-query tier never fires — the query is a token subset of the + label, not a prefix). The filler nodes make "judge"/"everything" common so + their IDF stays low. Decoy and target live in different components: resolving + the source to the decoy yields a false "No path found". + """ + nodes = [ + {"id": "target", "label": "Degenerate Reject-Everything Judge", "community": 0}, + {"id": "decoy", "label": "Rejection Summary", "community": 0}, + ] + for i in range(30): + nodes.append({"id": f"j{i}", "label": f"Judge Helper {i}", "community": 0}) + nodes.append({"id": f"e{i}", "label": f"Everything Widget {i}", "community": 0}) + graph_data = { + "directed": False, "multigraph": False, "graph": {}, + "nodes": nodes, + "links": [ + {"source": "target", "target": "j0", + "relation": "verified_by", "confidence": "EXTRACTED"}, + {"source": "decoy", "target": "e0", + "relation": "mentions", "confidence": "EXTRACTED"}, + ], + } + p = tmp_path / "graph.json" + p.write_text(json.dumps(graph_data)) + return p + + +def test_endpoint_prefers_full_token_match(monkeypatch, tmp_path, capsys): + """A token-subset query resolves to the full-match node, not the IDF head.""" + p = _write_misranking_graph(tmp_path) + out = _run(monkeypatch, p, "Reject-everything judge", "Judge Helper 0", capsys) + assert "Shortest path (1 hops):" in out + assert "Degenerate Reject-Everything Judge" in out + assert "No path found" not in out + + +def test_endpoint_falls_back_to_score_head(monkeypatch, tmp_path, capsys): + """No full-token candidate -> behavior identical to the old scored[0] pick.""" + p = _write_misranking_graph(tmp_path) + # "Rejection judge" full-matches nothing ("rejection" only appears in the + # decoy, "judge" never joins it), so the IDF head (the decoy) still wins, + # and the disconnected components make that a "No path found" exit(0). + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr(mainmod.sys, "argv", + ["graphify", "path", "Rejection judge", "Judge Helper 0", "--graph", str(p)]) + with pytest.raises(SystemExit) as exc_info: + mainmod.main() + assert exc_info.value.code == 0 + assert "No path found" in capsys.readouterr().out + + +# ── #2074: deterministic route + honest edge relation ──────────────────────── + +def _diamond_graph(tmp_path): + """Two equal-length routes A->P->B and A->Q->B — a tie the traversal must + resolve deterministically.""" + data = { + "directed": False, "multigraph": False, "graph": {}, + "nodes": [ + {"id": "a", "label": "Alpha", "source_file": "a.py"}, + {"id": "p", "label": "Pmid", "source_file": "p.py"}, + {"id": "q", "label": "Qmid", "source_file": "q.py"}, + {"id": "b", "label": "Beta", "source_file": "b.py"}, + ], + "links": [ + {"source": "a", "target": "p", "relation": "calls", "confidence": "EXTRACTED"}, + {"source": "p", "target": "b", "relation": "calls", "confidence": "EXTRACTED"}, + {"source": "a", "target": "q", "relation": "calls", "confidence": "EXTRACTED"}, + {"source": "q", "target": "b", "relation": "calls", "confidence": "EXTRACTED"}, + ], + } + p = tmp_path / "graph.json" + p.write_text(json.dumps(data)) + return p + + +def _arrow_line(stdout: str) -> str: + return next((l.strip() for l in stdout.splitlines() if "-->" in l or "<--" in l), "") + + +def test_path_deterministic_across_hash_seeds(tmp_path): + """#2074: the same graph must yield the same route regardless of + PYTHONHASHSEED. pytest fixes the seed per process, so run out-of-process.""" + gp = _diamond_graph(tmp_path) + routes = set() + for seed in ("0", "1", "2", "3", "4", "5", "6", "7"): + env = {**os.environ, "PYTHONHASHSEED": seed} + r = subprocess.run( + [sys.executable, "-m", "graphify", "path", "Alpha", "Beta", "--graph", str(gp)], + capture_output=True, text=True, env=env, cwd=str(tmp_path), + ) + assert r.returncode == 0, r.stderr + routes.add(_arrow_line(r.stdout)) + assert len(routes) == 1, f"non-deterministic path across hash seeds: {routes}" + # Canonical tie-break picks the lexicographically-smaller mid node (Pmid). + assert "Pmid" in next(iter(routes)) + + +def test_path_relation_matches_stored_edge_not_fabricated(monkeypatch, tmp_path, capsys): + """#2074: the printed relation must be the edge's ACTUAL stored relation, + never a hardcoded/fabricated `calls`.""" + data = { + "directed": False, "multigraph": False, "graph": {}, + "nodes": [ + {"id": "a", "label": "Alpha", "source_file": "a.py"}, + {"id": "b", "label": "Beta", "source_file": "b.py"}, + ], + "links": [ + {"source": "a", "target": "b", "relation": "references", "confidence": "INFERRED"}, + ], + } + gp = tmp_path / "graph.json" + gp.write_text(json.dumps(data)) + out = _run(monkeypatch, gp, "Alpha", "Beta", capsys) + assert "--references [INFERRED]-->" in out + assert "calls" not in out + + +def test_path_relation_fallback_related_when_missing(monkeypatch, tmp_path, capsys): + """#2074: an edge with no stored relation prints an honest 'related', not an + empty '---->' arrow and not a fabricated relation.""" + data = { + "directed": False, "multigraph": False, "graph": {}, + "nodes": [ + {"id": "a", "label": "Alpha", "source_file": "a.py"}, + {"id": "b", "label": "Beta", "source_file": "b.py"}, + ], + "links": [{"source": "a", "target": "b"}], + } + gp = tmp_path / "graph.json" + gp.write_text(json.dumps(data)) + out = _run(monkeypatch, gp, "Alpha", "Beta", capsys) + assert "--related-->" in out + assert "---->" not in out.replace("--related-->", "") diff --git a/tests/test_pg_introspect.py b/tests/test_pg_introspect.py index a133cdad3..e29e4374b 100644 --- a/tests/test_pg_introspect.py +++ b/tests/test_pg_introspect.py @@ -24,6 +24,8 @@ def _make_mock_psycopg(tables, views, routines, fks, ``connect_raises``, if set, is an exception *instance* raised by connect(). """ + executed_queries: list[str] = [] + class MockCursor: def __enter__(self): return self @@ -33,6 +35,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): def execute(self, query, params=None): self.query = query + executed_queries.append(query) def fetchall(self): q = self.query.strip().lower() @@ -42,7 +45,7 @@ def fetchall(self): return views elif "information_schema.routines" in q: return routines - elif "information_schema.referential_constraints" in q: + elif "pg_constraint" in q: return fks return [] @@ -75,6 +78,7 @@ def info(self): "host": host, "dbname": dbname, } + mock_psycopg._executed_queries = executed_queries return mock_psycopg @@ -244,6 +248,89 @@ def test_pg_introspect_composite_fk(): ) +def test_pg_introspect_fk_query_avoids_privilege_filtered_view(): + """#1746: information_schema.referential_constraints only shows constraints + where the current user has WRITE access to the referencing table (owner or + a privilege other than SELECT). A read-only introspection role therefore + gets zero FK rows — while tables/views/routines all still appear, since + SELECT is enough for those views — and the graph silently loses every + 'references' edge. The FK query must read pg_catalog.pg_constraint, which + is not privilege-filtered.""" + mock_tables = [ + ("public", "users", "BASE TABLE"), + ("public", "orders", "BASE TABLE"), + ] + mock_fks = [ + ("fk_orders_user_id", "public", "orders", ["user_id"], "public", "users", ["id"]), + ] + + mock_psycopg = _make_mock_psycopg(mock_tables, [], [], mock_fks) + + with patch.dict("sys.modules", {"psycopg": mock_psycopg}): + res = introspect_postgres("postgresql://readonly:secret@myhost/mydb") + + constraint_queries = [ + q for q in mock_psycopg._executed_queries if "constraint" in q.lower() + ] + assert constraint_queries, "no FK query was executed" + assert all( + "referential_constraints" not in q.lower() for q in constraint_queries + ), "FK query must not read information_schema.referential_constraints (privilege-filtered, #1746)" + assert any("pg_constraint" in q.lower() for q in constraint_queries) + + # And the FK still becomes a references edge end-to-end + ref_edges = [e for e in res["edges"] if e["relation"] == "references"] + assert len(ref_edges) == 1 + + +def test_pg_introspect_fk_edges_survive_unparseable_function_stubs(): + """#1854: FK edges must survive routines whose reconstructed DDL the SQL + grammar cannot parse. + + A C-language routine's "body" from information_schema.routines is just the + C symbol name, so its reconstructed stub — + CREATE FUNCTION "public"."levenshtein"() RETURNS void + AS $gfx$ levenshtein $gfx$ LANGUAGE c; + — is unparseable, and tree-sitter's error recovery consumes the statements + that follow it. With FK ALTER TABLEs emitted after the function DDL, every + 'references' edge was silently lost on any DB with a common extension + installed (uuid-ossp, pgcrypto, pg_trgm, fuzzystrmatch, …). FKs must be + emitted before the routine DDL so an unparseable stub can only damage + what follows it.""" + n = 6 + mock_tables = [("public", f"t{i}", "BASE TABLE") for i in range(n + 1)] + mock_views = [] + # One C-language extension routine (unparseable stub) + one plpgsql routine + mock_routines = [ + ("public", "levenshtein", "FUNCTION", "levenshtein", "C"), + ("public", "trigfunc", "FUNCTION", "BEGIN SELECT 1; END;", "PLPGSQL"), + ] + mock_fks = [ + (f"fk{i}", "public", f"t{i}", ["ref_id"], "public", f"t{i+1}", ["id"]) + for i in range(n) + ] + + mock_psycopg = _make_mock_psycopg(mock_tables, mock_views, mock_routines, mock_fks) + + with patch.dict("sys.modules", {"psycopg": mock_psycopg}): + res = introspect_postgres("postgresql://myuser:secret@myhost/mydb") + + errors = validate_extraction(res) + assert errors == [], f"Validation errors: {errors}" + + ids_to_labels = {node["id"]: node["label"] for node in res["nodes"]} + ref_pairs = { + (ids_to_labels[e["source"]], ids_to_labels[e["target"]]) + for e in res["edges"] + if e["relation"] == "references" + } + expected = {(_q("public", f"t{i}"), _q("public", f"t{i+1}")) for i in range(n)} + assert ref_pairs == expected, ( + f"FK edges lost to parser error recovery: expected {n}, " + f"got {len(ref_pairs)}: {sorted(ref_pairs)}" + ) + + def test_pg_introspect_connection_error(): """A psycopg.OperationalError must be re-raised as ConnectionError with a sanitized message (no DSN/credentials) and no stack-trace noise.""" @@ -274,8 +361,10 @@ class FakeOperationalError(Exception): def test_pg_introspect_import_error(): """If psycopg is missing, introspect_postgres raises ImportError.""" with patch.dict("sys.modules", {"psycopg": None}): - with pytest.raises(ImportError, match="psycopg is required"): + with pytest.raises(ImportError, match="psycopg is required") as exc_info: introspect_postgres("postgresql://localhost/db") + # #1906: the PyPI package is graphifyy (double-y), so the install hint must match + assert "graphifyy[postgres]" in str(exc_info.value) def test_pg_introspect_uri_forward_slashes(): diff --git a/tests/test_php_type_resolution.py b/tests/test_php_type_resolution.py new file mode 100644 index 000000000..0dff626d3 --- /dev/null +++ b/tests/test_php_type_resolution.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +from pathlib import Path + +from graphify.extract import extract + + +def _write(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def _node_by_id(result: dict, nid: str) -> dict | None: + return next((n for n in result["nodes"] if n.get("id") == nid), None) + + +def _class_defs(result: dict, label: str) -> list[dict]: + return [ + n for n in result["nodes"] + if n.get("label") == label and n.get("source_file") + ] + + +def test_php_external_namespaced_base_does_not_collapse_onto_internal_class(tmp_path: Path): + # #1923: `App\Models\Page` (internal) and `Filament\Pages\Page` (external, + # via `use`) share the simple name `Page`. The bare-name rewire must NOT + # collapse the external supertype reference onto the only internal `Page`. + model = _write( + tmp_path / "app/Models/Page.php", + " git subprocess runs. + completed = MagicMock(returncode=0, stdout="refs/remotes/origin/v8\n", stderr="") + with patch("graphify.prs._gh", return_value=None), \ + patch("subprocess.run", return_value=completed) as mock_run: + _detect_default_branch() + _args, kwargs = mock_run.call_args + assert kwargs.get("encoding") == "utf-8" diff --git a/tests/test_python_decorators.py b/tests/test_python_decorators.py new file mode 100644 index 000000000..9e7747ae2 --- /dev/null +++ b/tests/test_python_decorators.py @@ -0,0 +1,227 @@ +"""Regression tests: Python decorator references (#2154). + +Applying a Python decorator emitted no edge to the decorator symbol, so +`affected ` answered "No affected nodes found" for every function it +wraps — a silent false negative on reverse-impact queries. + +TS/JS already emitted these edges (`_ts_emit_decorator_edges`); the Python +`decorated_definition` branch walked its children only to propagate the parent +class id (#1050) and never looked at the `decorator` children. Python now emits +the same shape: `references` edges with context="decorator" from the decorated +function/class to the decorator symbol, resolved through the same +sourceless-stub path as type references so an imported decorator collapses onto +its real definition. +""" +from pathlib import Path + +from graphify.extract import _file_stem, _make_id, extract + + +def _write(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def _stem(file: str) -> str: + return _file_stem(Path(file)) + + +def _func_nid(file: str, func: str) -> str: + return _make_id(_stem(file), func) + + +def _class_nid(file: str, cls: str) -> str: + return _make_id(_stem(file), cls) + + +def _method_nid(file: str, cls: str, method: str) -> str: + return _make_id(_class_nid(file, cls), method) + + +def _deco_edges(result: dict, owner_nid: str) -> set[str]: + """Decorator-reference edge targets emitted from owner_nid.""" + return { + e["target"] + for e in result["edges"] + if e["source"] == owner_nid + and e["relation"] == "references" + and e.get("context") == "decorator" + } + + +def test_module_level_function_decorator(tmp_path): + # The issue repro: decorator imported from another module, applied to a + # module-level function. Target is the sourceless stub the rewire collapses. + f = _write(tmp_path / "pkg" / "consumer.py", + "from deco import my_decorator\n" + "\n" + "@my_decorator\n" + "def business_logic():\n" + " pass\n") + r = extract([f], cache_root=tmp_path) + assert _make_id("my_decorator") in _deco_edges( + r, _func_nid("pkg/consumer.py", "business_logic")) + + +def test_same_file_decorator_resolves_to_local_definition(tmp_path): + # Decorator defined above its use in the same file: the edge must point at + # the real local node, not a stub. + f = _write(tmp_path / "pkg" / "local.py", + "def my_decorator(fn):\n" + " return fn\n" + "\n" + "@my_decorator\n" + "def business_logic():\n" + " pass\n") + r = extract([f], cache_root=tmp_path) + assert _func_nid("pkg/local.py", "my_decorator") in _deco_edges( + r, _func_nid("pkg/local.py", "business_logic")) + + +def test_decorator_with_arguments(tmp_path): + # `@deco(arg)` is a `call` node; the head symbol is its `function` field. + f = _write(tmp_path / "pkg" / "args.py", + "from deco import retry\n" + "\n" + "@retry(times=3)\n" + "def flaky():\n" + " pass\n") + r = extract([f], cache_root=tmp_path) + assert _make_id("retry") in _deco_edges(r, _func_nid("pkg/args.py", "flaky")) + + +def test_attribute_decorator_targets_the_symbol_not_the_module(tmp_path): + # `@app.route("/")` is an `attribute` under a `call`; the target is `route`, + # matching _ts_decorator_name's member_expression handling. + f = _write(tmp_path / "pkg" / "web.py", + "import app\n" + "\n" + "@app.route(\"/\")\n" + "def index():\n" + " pass\n") + r = extract([f], cache_root=tmp_path) + targets = _deco_edges(r, _func_nid("pkg/web.py", "index")) + assert _make_id("route") in targets + assert _make_id("app") not in targets + + +def test_stacked_decorators_all_emit(tmp_path): + f = _write(tmp_path / "pkg" / "stack.py", + "from deco import a, b, c\n" + "\n" + "@a\n" + "@b\n" + "@c\n" + "def target():\n" + " pass\n") + r = extract([f], cache_root=tmp_path) + targets = _deco_edges(r, _func_nid("pkg/stack.py", "target")) + assert {_make_id("a"), _make_id("b"), _make_id("c")} <= targets + + +def test_decorated_method_owner_is_class_qualified(tmp_path): + # Guards the #1050 interaction: the owner must be the class-qualified method + # id, not a bare module-level id. + f = _write(tmp_path / "pkg" / "svc.py", + "from deco import traced\n" + "\n" + "class Service:\n" + " @traced\n" + " def handle(self):\n" + " pass\n") + r = extract([f], cache_root=tmp_path) + assert _make_id("traced") in _deco_edges( + r, _method_nid("pkg/svc.py", "Service", "handle")) + + +def test_property_still_class_qualified(tmp_path): + # #1050 regression guard: @property/@staticmethod must not change the + # method node's class-qualified id. + f = _write(tmp_path / "pkg" / "prop.py", + "class Config:\n" + " @property\n" + " def name(self):\n" + " return 1\n") + r = extract([f], cache_root=tmp_path) + assert any(n["id"] == _method_nid("pkg/prop.py", "Config", "name") + for n in r["nodes"]) + + +def test_decorated_class(tmp_path): + f = _write(tmp_path / "pkg" / "model.py", + "from registry import register_model\n" + "\n" + "@register_model\n" + "class Point:\n" + " x: int\n") + r = extract([f], cache_root=tmp_path) + assert _make_id("register_model") in _deco_edges( + r, _class_nid("pkg/model.py", "Point")) + + +def test_stdlib_class_decorator_emits_no_edge(tmp_path): + # @dataclass is ambient stdlib vocabulary (_PYTHON_DECORATOR_NOISE): no + # decorator edge and no sourceless `dataclass` stub node. + f = _write(tmp_path / "pkg" / "dc.py", + "from dataclasses import dataclass\n" + "\n" + "@dataclass\n" + "class Point:\n" + " x: int\n") + r = extract([f], cache_root=tmp_path) + assert _deco_edges(r, _class_nid("pkg/dc.py", "Point")) == set() + assert not any(n["id"] == _make_id("dataclass") for n in r["nodes"]) + + +def test_builtin_method_decorators_emit_no_edge_or_stub(tmp_path): + # @property / @staticmethod must not fabricate stub nodes or decorator + # edges — they would appear on nearly every class-heavy file. + f = _write(tmp_path / "pkg" / "builtins.py", + "class Config:\n" + " @property\n" + " def name(self):\n" + " return 1\n" + "\n" + " @staticmethod\n" + " def make():\n" + " return Config()\n") + r = extract([f], cache_root=tmp_path) + assert _deco_edges(r, _method_nid("pkg/builtins.py", "Config", "name")) == set() + assert _deco_edges(r, _method_nid("pkg/builtins.py", "Config", "make")) == set() + node_ids = {n["id"] for n in r["nodes"]} + assert _make_id("property") not in node_ids + assert _make_id("staticmethod") not in node_ids + + +def test_functools_wraps_does_not_rewire_onto_local_wraps(tmp_path): + # The demonstrated false positive: a corpus defining its own top-level + # `def wraps(...)` while another file uses `@functools.wraps` got a false + # decorator edge onto the local `wraps` via the unique-function rewire. + _write(tmp_path / "pkg" / "gift.py", + "def wraps(thing):\n" + " return thing\n") + f = _write(tmp_path / "pkg" / "util.py", + "import functools\n" + "\n" + "def logged(fn):\n" + " @functools.wraps(fn)\n" + " def inner(*args, **kwargs):\n" + " return fn(*args, **kwargs)\n" + " return inner\n") + r = extract([tmp_path / "pkg" / "gift.py", f], cache_root=tmp_path) + local_wraps = _func_nid("pkg/gift.py", "wraps") + assert not any( + e["target"] == local_wraps + and e["relation"] == "references" + and e.get("context") == "decorator" + for e in r["edges"] + ) + + +def test_undecorated_function_emits_no_decorator_edge(tmp_path): + f = _write(tmp_path / "pkg" / "plain.py", + "def plain():\n" + " pass\n") + r = extract([f], cache_root=tmp_path) + assert _deco_edges(r, _func_nid("pkg/plain.py", "plain")) == set() diff --git a/tests/test_query_cli.py b/tests/test_query_cli.py index cf8eb6e56..0db4e6fa8 100644 --- a/tests/test_query_cli.py +++ b/tests/test_query_cli.py @@ -51,6 +51,61 @@ def test_query_cli_heuristic_context_filter(monkeypatch, tmp_path, capsys): assert "build" not in out +def _write_calls_graph(tmp_path): + """A single directed `calls` edge on an (on-disk) undirected graph.json, + + the standard `graphify extract`/`update` output shape (`"directed": + false`, direction implied only by each link's source/target). + """ + G = nx.Graph() + G.add_node("caller", label="caller_fn", source_file="a.py", source_location="L1", community=0) + G.add_node("callee", label="callee_fn", source_file="b.py", source_location="L1", community=1) + G.add_edge("caller", "callee", relation="calls", confidence="EXTRACTED", context="call") + graph_path = tmp_path / "graph.json" + graph_path.write_text(json.dumps(json_graph.node_link_data(G, edges="links"))) + return graph_path + + +def test_query_cli_preserves_calls_direction_when_seeded_on_callee(monkeypatch, tmp_path, capsys): + """`graphify query` must render `calls` edges caller->callee regardless of + which endpoint the query term matches first. + + The graph `query` loads is undirected (so BFS/DFS can explore both + callers and callees of the seed), so `G.neighbors()` returns `caller_fn` + as a neighbor of `callee_fn` with no direction of its own. Before the + fix, the renderer assumed the BFS/DFS visit order (u, v) was the edge's + (source, target), so seeding on the callee printed the edge backwards: + "callee_fn --calls--> caller_fn". graph.json's `source`/`target` for this + edge stay correct on disk either way; only the query rendering was wrong. + """ + graph_path = _write_calls_graph(tmp_path) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, + "argv", + ["graphify", "query", "callee_fn", "--graph", str(graph_path)], + ) + mainmod.main() + out = capsys.readouterr().out + assert "caller_fn --calls" in out + assert "callee_fn --calls" not in out + + +def test_query_cli_preserves_calls_direction_when_seeded_on_caller(monkeypatch, tmp_path, capsys): + """Same edge, seeded from the caller side — must stay correct too.""" + graph_path = _write_calls_graph(tmp_path) + monkeypatch.setattr(mainmod, "_check_skill_version", lambda _: None) + monkeypatch.setattr( + mainmod.sys, + "argv", + ["graphify", "query", "caller_fn", "--graph", str(graph_path)], + ) + mainmod.main() + out = capsys.readouterr().out + assert "caller_fn --calls" in out + assert "callee_fn --calls" not in out + + def test_query_cli_rejects_oversized_graph(monkeypatch, tmp_path, capsys): """#F4: query CLI must refuse to parse a graph.json that exceeds the cap.""" import pytest diff --git a/tests/test_querylog.py b/tests/test_querylog.py index 2ebe851e7..b843550c7 100644 --- a/tests/test_querylog.py +++ b/tests/test_querylog.py @@ -176,3 +176,48 @@ def test_kind_mcp_query(tmp_path, monkeypatch): rec = json.loads(log_file.read_text()) assert rec["kind"] == "mcp_query" + + +# --------------------------------------------------------------------------- +# #1797 — query log is opt-in (default OFF) +# --------------------------------------------------------------------------- + +def _clear_log_env(monkeypatch): + for k in ("GRAPHIFY_QUERY_LOG", "GRAPHIFY_QUERY_LOG_ENABLE", "GRAPHIFY_QUERY_LOG_DISABLE"): + monkeypatch.delenv(k, raising=False) + + +def test_query_log_off_by_default(monkeypatch): + from graphify.querylog import _log_path + _clear_log_env(monkeypatch) + assert _log_path() is None + + +def test_query_log_enabled_by_explicit_flag(monkeypatch): + from graphify.querylog import _log_path + _clear_log_env(monkeypatch) + monkeypatch.setenv("GRAPHIFY_QUERY_LOG_ENABLE", "1") + assert str(_log_path()).endswith("graphify-queries.log") + + +def test_query_log_enabled_by_explicit_path(monkeypatch, tmp_path): + from graphify.querylog import _log_path + _clear_log_env(monkeypatch) + monkeypatch.setenv("GRAPHIFY_QUERY_LOG", str(tmp_path / "q.log")) + assert _log_path() == tmp_path / "q.log" + + +def test_query_log_disable_wins(monkeypatch): + from graphify.querylog import _log_path + _clear_log_env(monkeypatch) + monkeypatch.setenv("GRAPHIFY_QUERY_LOG_ENABLE", "1") + monkeypatch.setenv("GRAPHIFY_QUERY_LOG_DISABLE", "1") + assert _log_path() is None + + +def test_log_query_writes_nothing_by_default(monkeypatch, tmp_path): + """End-to-end: with no opt-in, log_query must not create the default log.""" + _clear_log_env(monkeypatch) + monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path)) + log_query(kind="query", question="secret internal ticket TICKET-123", corpus=".", result="1 node found") + assert not (tmp_path / ".cache" / "graphify-queries.log").exists() diff --git a/tests/test_read_hook.py b/tests/test_read_hook.py index d3bf58a31..82a514d92 100644 --- a/tests/test_read_hook.py +++ b/tests/test_read_hook.py @@ -1,17 +1,31 @@ -"""The Read|Glob PreToolUse hook nudges toward the graph instead of raw reads. +"""The Read|Glob PreToolUse guard nudges toward the graph instead of raw reads. Closes the issue #1114 gap: the Bash search hook never sees a file read through -the native Read tool or a Glob. These tests run the hook command the way Claude -Code does - via `sh -c` with crafted stdin JSON - and assert it nudges only for -a source/doc file outside graphify-out/ when a graph exists, and otherwise stays -silent and fails open. +the native Read tool or a Glob. Since #522 the guard runs as the shell-agnostic +`graphify hook-guard read` subcommand (not inline bash), so it works on Windows +too. These tests invoke that subcommand with crafted stdin JSON and assert it +nudges only for a source/doc file outside graphify-out/ when a graph exists, and +otherwise stays silent and fails open. """ import json +import os import subprocess +import sys -from graphify.__main__ import _READ_SETTINGS_HOOK +from graphify.__main__ import _claude_pretooluse_hooks -CMD = _READ_SETTINGS_HOOK["hooks"][0]["command"] + +def _read_matcher(): + hooks = _claude_pretooluse_hooks() + return next(h for h in hooks if h["matcher"] == "Read|Glob") + + +def _env(): + # The guard resolves the graph via GRAPHIFY_OUT (default "graphify-out", + # relative to cwd). Drop any inherited override so the tmp_path graph is found. + e = dict(os.environ) + e.pop("GRAPHIFY_OUT", None) + return e def _run(tool_input, cwd, *, graph: bool): @@ -20,12 +34,21 @@ def _run(tool_input, cwd, *, graph: bool): (cwd / "graphify-out" / "graph.json").write_text("{}", encoding="utf-8") stdin = json.dumps({"tool_input": tool_input}) return subprocess.run( - ["sh", "-c", CMD], input=stdin, capture_output=True, text=True, cwd=cwd + [sys.executable, "-m", "graphify", "hook-guard", "read"], + input=stdin, capture_output=True, text=True, cwd=cwd, env=_env(), ) def test_matcher_targets_read_and_glob(): - assert _READ_SETTINGS_HOOK["matcher"] == "Read|Glob" + assert _read_matcher()["matcher"] == "Read|Glob" + + +def test_command_has_no_shell_syntax(): + # #522: the command must be a plain exe invocation, not POSIX bash. + cmd = _read_matcher()["hooks"][0]["command"] + for token in ("$(", "case ", "[ -f", "&&", "||", ";;", "echo '"): + assert token not in cmd, f"shell syntax {token!r} leaked into the hook" + assert "graphify" in cmd and "hook-guard read" in cmd def test_silent_without_graph(tmp_path): @@ -106,14 +129,15 @@ def test_fails_open_on_malformed_stdin(tmp_path): (tmp_path / "graphify-out").mkdir() (tmp_path / "graphify-out" / "graph.json").write_text("{}", encoding="utf-8") r = subprocess.run( - ["sh", "-c", CMD], input="this is not json", capture_output=True, text=True, cwd=tmp_path + [sys.executable, "-m", "graphify", "hook-guard", "read"], + input="this is not json", capture_output=True, text=True, cwd=tmp_path, env=_env(), ) assert r.returncode == 0 assert r.stdout.strip() == "" def test_never_blocks(tmp_path): - """A nudge is additionalContext only - the hook must exit 0, never deny.""" + """A nudge is additionalContext only - the guard must exit 0, never deny.""" r = _run({"file_path": "src/app.py"}, tmp_path, graph=True) assert r.returncode == 0 assert '"permissionDecision"' not in r.stdout diff --git a/tests/test_replace_or_append_section.py b/tests/test_replace_or_append_section.py new file mode 100644 index 000000000..f098a0070 --- /dev/null +++ b/tests/test_replace_or_append_section.py @@ -0,0 +1,62 @@ +"""#1688 - graphify's shared-file section update must not destroy user content. + +_replace_or_append_section used to locate its marker (`## graphify`) as a +substring, so a bullet or inline reference to the section became the replace +anchor and every line from there to the next heading was deleted. The marker is +now matched only as an exact heading line. +""" +from __future__ import annotations + +from graphify.__main__ import _replace_or_append_section + +MARKER = "## graphify" +NEW = "## graphify\n\nThis project has a knowledge graph at graphify-out/.\n" + + +def test_inline_reference_to_marker_is_not_treated_as_the_section(): + before = ( + "# My Project\n\n" + "## Setup\n" + "- See the `## graphify` section for graph usage.\n\n" + "## Release Process\n" + "Critical steps that must not be lost.\n" + ) + after = _replace_or_append_section(before, MARKER, NEW) + assert "See the `## graphify` section" in after # bullet preserved + assert "Critical steps that must not be lost" in after # later section preserved + assert "knowledge graph at graphify-out/" in after # section still added + + +def test_real_section_is_replaced_in_place(): + before = ( + "# P\n\n## Setup\n- do things\n\n" + "## graphify\n\nOLD text.\n\n" + "## Release\nkeep me\n" + ) + after = _replace_or_append_section(before, MARKER, NEW) + assert "OLD text." not in after + assert "knowledge graph at graphify-out/" in after + assert "do things" in after and "keep me" in after + + +def test_reinstall_is_idempotent(): + once = _replace_or_append_section("# P\n\n## Setup\n- x\n", MARKER, NEW) + twice = _replace_or_append_section(once, MARKER, NEW) + assert once.split("\n").count(MARKER) == 1 + assert twice.split("\n").count(MARKER) == 1 + + +def test_append_when_no_real_heading(): + before = "# P\n\n## Setup\n- x\n" + after = _replace_or_append_section(before, MARKER, NEW) + assert "- x" in after + assert after.split("\n").count(MARKER) == 1 + + +def test_prefers_last_heading_when_duplicated(): + before = "## graphify\nstale early copy\n\n## Other\nmid\n\n## graphify\nreal trailing copy\n" + after = _replace_or_append_section(before, MARKER, NEW) + # the trailing real section is replaced; the earlier stray heading + the + # user's "mid" content are left intact + assert "mid" in after + assert "knowledge graph at graphify-out/" in after diff --git a/tests/test_report.py b/tests/test_report.py index 00be0f36d..767e2ba34 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -135,3 +135,22 @@ def test_import_cycles_section_absent_for_documents_only_corpus(): tokens = {"input": 0, "output": 0} report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project") assert "## Import Cycles" not in report + + +def test_report_hubs_are_plain_text_by_default(): + # #1712: without --obsidian the _COMMUNITY_*.md notes don't exist, so wikilinks + # would dangle (and pollute an Obsidian vault's graph view). Default to plain text. + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + labels = {cid: f"Widget {cid}" for cid in communities} + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project", min_community_size=1) + assert "## Community Hubs (Navigation)" in report + assert "[[_COMMUNITY_" not in report, "must not emit dangling Obsidian wikilinks by default (#1712)" + assert any(f"- Widget {cid}" in report for cid in communities) + + +def test_report_hubs_use_wikilinks_when_obsidian(): + # The opt-in path keeps the vault-navigable wikilink form. + G, communities, cohesion, labels, gods, surprises, detection, tokens = make_inputs() + labels = {cid: f"Widget {cid}" for cid in communities} + report = generate(G, communities, cohesion, labels, gods, surprises, detection, tokens, "./project", min_community_size=1, obsidian=True) + assert "[[_COMMUNITY_" in report diff --git a/tests/test_ruby_resolution.py b/tests/test_ruby_resolution.py index 5cd1b1150..7d402524b 100644 --- a/tests/test_ruby_resolution.py +++ b/tests/test_ruby_resolution.py @@ -348,3 +348,21 @@ def test_mixin_is_not_emitted_as_calls_edge(tmp_path: Path) -> None: for e in g["edges"] if e.get("relation") == "calls"} assert ("K", "C") not in calls assert ("K", "C") in _mixes_in(g) + + +def test_rake_files_extract_and_resolve_like_rb(tmp_path): + """#1784: `.rake` files are plain Ruby and must route to the Ruby extractor + and participate in Ruby cross-file resolution exactly like `.rb`.""" + rake = _write(tmp_path, "ops.rake", + "class RakeHelper\n def self.run\n Widget.tally\n end\nend\n") + rb = _write(tmp_path, "widget.rb", + "class Widget\n def self.tally\n 42\n end\nend\n") + result = extract([rake, rb], cache_root=tmp_path / ".cache") + label = {n["id"]: n.get("label") for n in result["nodes"]} + labels = set(label.values()) + # the .rake file's symbols are extracted + assert "RakeHelper" in labels and ".run()" in labels + # and the cross-file member call resolves .rake -> .rb + calls = {(label.get(e["source"]), label.get(e["target"])) + for e in result["edges"] if e["relation"] == "calls"} + assert (".run()", ".tally()") in calls diff --git a/tests/test_search_hook.py b/tests/test_search_hook.py new file mode 100644 index 000000000..93c492375 --- /dev/null +++ b/tests/test_search_hook.py @@ -0,0 +1,185 @@ +"""The Bash PreToolUse guard nudges toward the graph before grep/find searches. + +Since #522 it runs as the shell-agnostic `graphify hook-guard search` subcommand +(not inline bash), so it works on Windows too. These tests invoke the subcommand +with crafted stdin JSON and assert it nudges only for a search command when a +graph exists, and otherwise stays silent and fails open. +""" +import json +import os +import subprocess +import sys + +from graphify.__main__ import _claude_pretooluse_hooks + + +def _search_matcher(): + hooks = _claude_pretooluse_hooks() + return next(h for h in hooks if h["matcher"] == "Bash|Grep") + + +def _env(): + e = dict(os.environ) + e.pop("GRAPHIFY_OUT", None) + return e + + +def _run(command, cwd, *, graph: bool): + if graph: + (cwd / "graphify-out").mkdir(parents=True, exist_ok=True) + (cwd / "graphify-out" / "graph.json").write_text("{}", encoding="utf-8") + stdin = json.dumps({"tool_input": {"command": command}}) + return subprocess.run( + [sys.executable, "-m", "graphify", "hook-guard", "search"], + input=stdin, capture_output=True, text=True, cwd=cwd, env=_env(), + ) + + +def _run_grep_tool(tool_input, cwd, *, graph: bool): + """Feed a Grep-tool-shaped payload (pattern/path/glob, no command) to the guard.""" + if graph: + (cwd / "graphify-out").mkdir(parents=True, exist_ok=True) + (cwd / "graphify-out" / "graph.json").write_text("{}", encoding="utf-8") + stdin = json.dumps({"tool_name": "Grep", "tool_input": tool_input}) + return subprocess.run( + [sys.executable, "-m", "graphify", "hook-guard", "search"], + input=stdin, capture_output=True, text=True, cwd=cwd, env=_env(), + ) + + +def test_matcher_targets_bash_and_grep(): + # #1986: content search goes through Claude Code's dedicated Grep tool, so + # the matcher must cover it alongside Bash. + assert _search_matcher()["matcher"] == "Bash|Grep" + + +def test_hook_command_has_no_backslashes(monkeypatch): + # On Windows the resolved exe is a backslash path; Claude Code runs command + # hooks through Git Bash by default, which treats an unquoted backslash as an + # escape character and strips it (C:\Users\me\graphify.EXE -> C:Usersme...), + # breaking every guard. The emitted command must use forward slashes. + from graphify.__main__ import _resolve_graphify_exe + monkeypatch.setattr("shutil.which", lambda _name: r"C:\Users\me\graphify.EXE") + assert _resolve_graphify_exe() == "C:/Users/me/graphify.EXE" + for h in _claude_pretooluse_hooks(): + assert "\\" not in h["hooks"][0]["command"] + + +def test_command_has_no_shell_syntax(): + # #522: no POSIX bash that Windows cmd.exe/PowerShell can't parse. + cmd = _search_matcher()["hooks"][0]["command"] + for token in ("$(", "case ", "[ -f", "&&", "||", ";;", "echo '"): + assert token not in cmd, f"shell syntax {token!r} leaked into the hook" + assert "graphify" in cmd and "hook-guard search" in cmd + + +def test_nudges_on_search_commands_with_graph(tmp_path): + for command in ( + "grep -rn foo .", + "rg pattern src/", + "ripgrep thing", + "find . -name '*.py'", + "fd bar", + "ack needle", + "ag needle", + ): + out = _run(command, tmp_path, graph=True).stdout + assert "graphify query" in out, f"{command!r} should nudge" + + +def test_silent_without_graph(tmp_path): + out = _run("grep -rn foo .", tmp_path, graph=False).stdout + assert out.strip() == "" + + +def test_silent_on_non_search_commands(tmp_path): + for command in ("ls -la", "git status", "cat README.md", "python app.py"): + out = _run(command, tmp_path, graph=True).stdout + assert out.strip() == "", f"{command!r} should not nudge" + + +def test_nudge_payload_is_valid_pretooluse_json(tmp_path): + out = _run("grep -rn foo .", tmp_path, graph=True).stdout + payload = json.loads(out) + assert payload["hookSpecificOutput"]["hookEventName"] == "PreToolUse" + assert "graphify query" in payload["hookSpecificOutput"]["additionalContext"] + + +def test_fails_open_on_malformed_stdin(tmp_path): + (tmp_path / "graphify-out").mkdir() + (tmp_path / "graphify-out" / "graph.json").write_text("{}", encoding="utf-8") + r = subprocess.run( + [sys.executable, "-m", "graphify", "hook-guard", "search"], + input="not json", capture_output=True, text=True, cwd=tmp_path, env=_env(), + ) + assert r.returncode == 0 + assert r.stdout.strip() == "" + + +def test_never_blocks(tmp_path): + r = _run("grep -rn foo .", tmp_path, graph=True) + assert r.returncode == 0 + assert '"permissionDecision"' not in r.stdout + assert '"deny"' not in r.stdout + + +def test_honors_graphify_out_override(tmp_path): + """The guard resolves the graph via GRAPHIFY_OUT, not a hardcoded path.""" + custom = tmp_path / "custom-out" + custom.mkdir() + (custom / "graph.json").write_text("{}", encoding="utf-8") + env = dict(os.environ, GRAPHIFY_OUT=str(custom)) + stdin = json.dumps({"tool_input": {"command": "grep -rn foo ."}}) + r = subprocess.run( + [sys.executable, "-m", "graphify", "hook-guard", "search"], + input=stdin, capture_output=True, text=True, cwd=tmp_path, env=env, + ) + assert "graphify query" in r.stdout + + +# --------------------------------------------------------------------------- +# #1986: the dedicated Grep tool (pattern/path/glob, no command) must nudge too +# --------------------------------------------------------------------------- + + +def test_grep_tool_input_nudges_with_graph(tmp_path): + for tool_input in ( + {"pattern": "extract_corpus", "path": "."}, + {"pattern": "TODO"}, + {"pattern": "def main", "glob": "*.py"}, + {"pattern": "foo", "path": "src/", "glob": "**/*.ts"}, + ): + out = _run_grep_tool(tool_input, tmp_path, graph=True).stdout + assert "graphify query" in out, f"Grep input {tool_input!r} should nudge" + + +def test_grep_tool_input_silent_without_graph(tmp_path): + out = _run_grep_tool({"pattern": "foo", "path": "."}, tmp_path, graph=False).stdout + assert out.strip() == "" + + +def test_grep_tool_nudge_is_valid_pretooluse_json(tmp_path): + out = _run_grep_tool({"pattern": "foo", "path": "."}, tmp_path, graph=True).stdout + payload = json.loads(out) + assert payload["hookSpecificOutput"]["hookEventName"] == "PreToolUse" + assert "graphify query" in payload["hookSpecificOutput"]["additionalContext"] + + +def test_grep_tool_never_blocks(tmp_path): + r = _run_grep_tool({"pattern": "foo", "path": "."}, tmp_path, graph=True) + assert r.returncode == 0 + assert '"permissionDecision"' not in r.stdout + assert '"deny"' not in r.stdout + + +def test_bash_non_search_with_stray_pattern_key_does_not_nudge(tmp_path): + """A Bash tool_input carries `command`; the Grep-shape detection must not + fire when a command is present but is not a search.""" + (tmp_path / "graphify-out").mkdir(parents=True, exist_ok=True) + (tmp_path / "graphify-out" / "graph.json").write_text("{}", encoding="utf-8") + stdin = json.dumps({"tool_input": {"command": "ls -la", "pattern": "x"}}) + r = subprocess.run( + [sys.executable, "-m", "graphify", "hook-guard", "search"], + input=stdin, capture_output=True, text=True, cwd=tmp_path, env=_env(), + ) + assert r.stdout.strip() == "" diff --git a/tests/test_security.py b/tests/test_security.py index d2e08c0d0..74669f938 100644 --- a/tests/test_security.py +++ b/tests/test_security.py @@ -20,6 +20,7 @@ _MAX_FETCH_BYTES, _MAX_GRAPH_FILE_BYTES, _MAX_TEXT_BYTES, + _max_graph_file_bytes, _METADATA_MAX_LIST_ITEMS, _METADATA_MAX_VALUE_LEN, _sanitize_metadata_string, @@ -218,6 +219,12 @@ def test_sanitize_label_safe_passthrough(): assert sanitize_label("MyClass") == "MyClass" assert sanitize_label("extract_python") == "extract_python" +def test_sanitize_label_none_returns_empty(): + # #1775: a node with source_file=None / label=None (synthetic/aggregate + # nodes, or JSON `null`) must not raise — .get() returns None, not the + # default, when the key is present-but-null. + assert sanitize_label(None) == "" + # --------------------------------------------------------------------------- # check_graph_file_size_cap (#F4 — graph-load memory bomb protection) @@ -227,6 +234,57 @@ def test_graph_size_cap_default_is_512_mib(): assert _MAX_GRAPH_FILE_BYTES == 512 * 1024 * 1024 +# --------------------------------------------------------------------------- +# _max_graph_file_bytes — GRAPHIFY_MAX_GRAPH_BYTES env-var parsing +# --------------------------------------------------------------------------- + +def test_max_graph_bytes_default_when_unset(monkeypatch): + monkeypatch.delenv("GRAPHIFY_MAX_GRAPH_BYTES", raising=False) + assert _max_graph_file_bytes() == _MAX_GRAPH_FILE_BYTES + + +def test_max_graph_bytes_default_when_blank(monkeypatch): + monkeypatch.setenv("GRAPHIFY_MAX_GRAPH_BYTES", " ") + assert _max_graph_file_bytes() == _MAX_GRAPH_FILE_BYTES + + +def test_max_graph_bytes_plain_integer(monkeypatch): + monkeypatch.setenv("GRAPHIFY_MAX_GRAPH_BYTES", "671088640") + assert _max_graph_file_bytes() == 671088640 + + +def test_max_graph_bytes_mb_suffix_is_binary(monkeypatch): + monkeypatch.setenv("GRAPHIFY_MAX_GRAPH_BYTES", "640MB") + assert _max_graph_file_bytes() == 640 * 1024 * 1024 + + +def test_max_graph_bytes_gb_suffix_is_binary(monkeypatch): + monkeypatch.setenv("GRAPHIFY_MAX_GRAPH_BYTES", "2GB") + assert _max_graph_file_bytes() == 2 * 1024 * 1024 * 1024 + + +def test_max_graph_bytes_suffix_is_case_insensitive(monkeypatch): + monkeypatch.setenv("GRAPHIFY_MAX_GRAPH_BYTES", "3gb") + assert _max_graph_file_bytes() == 3 * 1024 * 1024 * 1024 + + +def test_max_graph_bytes_tolerates_space_before_suffix(monkeypatch): + monkeypatch.setenv("GRAPHIFY_MAX_GRAPH_BYTES", "5 GB") + assert _max_graph_file_bytes() == 5 * 1024 * 1024 * 1024 + + +@pytest.mark.parametrize("bad", ["not-a-number", "1.5GB", "0x10", "640KB"]) +def test_max_graph_bytes_unparseable_falls_back(monkeypatch, bad): + monkeypatch.setenv("GRAPHIFY_MAX_GRAPH_BYTES", bad) + assert _max_graph_file_bytes() == _MAX_GRAPH_FILE_BYTES + + +@pytest.mark.parametrize("nonpositive", ["0", "-1", "-4GB"]) +def test_max_graph_bytes_nonpositive_falls_back(monkeypatch, nonpositive): + monkeypatch.setenv("GRAPHIFY_MAX_GRAPH_BYTES", nonpositive) + assert _max_graph_file_bytes() == _MAX_GRAPH_FILE_BYTES + + def test_graph_size_cap_under_limit_returns_none(tmp_path): p = tmp_path / "graph.json" p.write_text('{"nodes": [], "links": []}', encoding="utf-8") diff --git a/tests/test_semantic_cache_out_root.py b/tests/test_semantic_cache_out_root.py new file mode 100644 index 000000000..2bde8d3c9 --- /dev/null +++ b/tests/test_semantic_cache_out_root.py @@ -0,0 +1,221 @@ +"""Regression tests for #1990 and #1991. + +#1990 — `graphify extract --out` saves recovery checkpoints in the wrong directory. + save_semantic_cache must write to cache_root (the --out dir), not root + (the corpus dir), when they differ. + +#1991 — Final semantic-cache save resolves source_file paths against out_root + and silently writes 0 entries when corpus files do not exist there. + Fixed by passing root=target (corpus), cache_root=out_root to + save_semantic_cache so source_file resolution and cache placement are + independently anchored. +""" +import warnings +from pathlib import Path + +import pytest + +from graphify.cache import ( + check_semantic_cache, + file_hash, + load_cached, + save_semantic_cache, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _semantic_dir(root: Path, mode: str | None = None) -> Path: + kind = "semantic" if mode is None else f"semantic-{mode}" + return root / "graphify-out" / "cache" / kind + + +def _count_cache_files(base: Path) -> int: + """Count .json files under a cache dir (recursively, excluding .tmp).""" + if not base.is_dir(): + return 0 + return sum(1 for _ in base.rglob("*.json")) + + +# --------------------------------------------------------------------------- +# #1990 — checkpoint writes to cache_root, not corpus root +# --------------------------------------------------------------------------- + + +def test_save_semantic_cache_writes_to_cache_root_not_corpus(tmp_path): + """When cache_root differs from root, cache files must land under cache_root.""" + corpus = tmp_path / "corpus" + out = tmp_path / "out" + corpus.mkdir() + out.mkdir() + + doc = corpus / "report.md" + doc.write_text("# Report\nSome content here.") + + nodes = [{"id": "n1", "label": "Report", "source_file": str(doc)}] + edges: list = [] + + saved = save_semantic_cache( + nodes, edges, root=corpus, cache_root=out + ) + + assert saved == 1, "expected 1 entry written" + # Cache file must be under the out dir, not the corpus dir + assert _count_cache_files(_semantic_dir(out)) == 1 + assert _count_cache_files(_semantic_dir(corpus)) == 0 + + +def test_save_semantic_cache_no_corpus_graphify_out_created(tmp_path): + """With cache_root set, no graphify-out/ dir should be created inside corpus.""" + corpus = tmp_path / "corpus" + out = tmp_path / "out" + corpus.mkdir() + out.mkdir() + + doc = corpus / "notes.md" + doc.write_text("Notes content.") + + save_semantic_cache( + [{"id": "x", "label": "X", "source_file": str(doc)}], + [], + root=corpus, + cache_root=out, + ) + + assert not (corpus / "graphify-out").exists(), ( + "graphify-out/ must not be created inside the corpus when cache_root is set" + ) + + +# --------------------------------------------------------------------------- +# #1990 — checkpoint written with cache_root is found on recovery read +# --------------------------------------------------------------------------- + + +def test_checkpoint_with_cache_root_is_found_by_check_semantic_cache(tmp_path): + """A checkpoint saved with cache_root can be retrieved by check_semantic_cache + using the same root/cache_root split — simulating recovery after interruption.""" + corpus = tmp_path / "corpus" + out = tmp_path / "out" + corpus.mkdir() + out.mkdir() + + doc = corpus / "paper.md" + doc.write_text("Some academic content.") + + nodes = [{"id": "p1", "label": "Paper", "source_file": str(doc)}] + + # Simulate a checkpoint written mid-run (merge_existing path) + save_semantic_cache( + nodes, [], + root=corpus, + cache_root=out, + merge_existing=True, + allowed_source_files=[doc], + ) + + # Recovery read: check_semantic_cache with the same root/cache_root split + # cli.py now uses (root=target, cache_root=out_root). + cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache( + [str(doc)], root=corpus, cache_root=out + ) + + assert len(uncached) == 0, f"expected cache hit, got miss for: {uncached}" + assert any(n.get("id") == "p1" for n in cached_nodes) + + +# --------------------------------------------------------------------------- +# #1991 — final save resolves source_file against corpus root, not out_root +# --------------------------------------------------------------------------- + + +def test_final_save_with_out_root_populates_cache(tmp_path): + """When root=corpus and cache_root=out, source_file resolution must use + corpus as anchor so p.is_file() succeeds and the entry is written.""" + corpus = tmp_path / "corpus" + out = tmp_path / "out" + corpus.mkdir() + out.mkdir() + + doc = corpus / "report.md" + doc.write_text("# Annual Report\nKey findings.") + + # This is the pattern that was broken: source_file is relative to corpus, + # but was previously resolved against out_root, making p.is_file() → False. + nodes = [{"id": "r1", "label": "AnnualReport", "source_file": "report.md"}] + + saved = save_semantic_cache( + nodes, [], + root=corpus, + cache_root=out, + allowed_source_files=[doc], + ) + + assert saved == 1, ( + "final save must write 1 entry when root=corpus and source_file is " + "relative to corpus — resolving against out_root caused 0 writes (#1991)" + ) + assert _count_cache_files(_semantic_dir(out)) == 1 + + +def test_final_save_with_wrong_root_emits_warning(tmp_path): + """Passing root=out_root (the old broken behaviour) silently writes 0 entries; + the fix adds a RuntimeWarning when ALL groups are dropped.""" + corpus = tmp_path / "corpus" + out = tmp_path / "out" + corpus.mkdir() + out.mkdir() + + doc = corpus / "report.md" + doc.write_text("# Report") + + # Old (broken) call: root=out, no cache_root — the corpus-relative + # source_file is resolved against out where the file doesn't exist + # → 0 entries written. + nodes = [{"id": "r1", "label": "R", "source_file": "report.md"}] + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + saved = save_semantic_cache(nodes, [], root=out) + + assert saved == 0 + # The new warning must fire when every group is dropped + assert any( + "#1991" in str(warning.message) for warning in w + ), "expected RuntimeWarning mentioning #1991 when all groups are silently skipped" + + +# --------------------------------------------------------------------------- +# Backward compat — omitting cache_root keeps legacy behaviour +# --------------------------------------------------------------------------- + + +def test_save_semantic_cache_backward_compat_no_cache_root(tmp_path): + """When cache_root is omitted, cache files still land under root (unchanged).""" + root = tmp_path / "project" + root.mkdir() + + doc = root / "main.md" + doc.write_text("Main content.") + + nodes = [{"id": "m1", "label": "Main", "source_file": str(doc)}] + + saved = save_semantic_cache(nodes, [], root=root) + + assert saved == 1 + assert _count_cache_files(_semantic_dir(root)) == 1 + + +def test_extract_corpus_parallel_accepts_cache_root_kwarg(): + """extract_corpus_parallel must accept a cache_root kwarg without raising + (import + signature check — no actual LLM call).""" + import inspect + from graphify.llm import extract_corpus_parallel + + sig = inspect.signature(extract_corpus_parallel) + assert "cache_root" in sig.parameters, ( + "extract_corpus_parallel must expose cache_root so cli.py can plumb " + "out_root through to _checkpoint_chunk (#1990)" + ) diff --git a/tests/test_semantic_cleanup.py b/tests/test_semantic_cleanup.py index 49fd03398..de13bd245 100644 --- a/tests/test_semantic_cleanup.py +++ b/tests/test_semantic_cleanup.py @@ -51,11 +51,15 @@ def test_validate_semantic_fragment_rejects_path_separator_in_id(): assert any("nodes[0].id" in e for e in errors) -def test_validate_semantic_fragment_rejects_invalid_file_type(): +def test_validate_semantic_fragment_accepts_unknown_file_type(): + """An unknown/synonym file_type is NOT a validation failure: build_from_json + coerces any value via _FILE_TYPE_SYNONYMS (unknown -> "concept", #840), so + rejecting a whole chunk over it would be pure data loss. file_type carries no + security risk, so it is left to build's coercion rather than gated here.""" fragment = _valid_fragment() fragment["nodes"][0]["file_type"] = "executable" errors = sc.validate_semantic_fragment(fragment) - assert any("file_type" in e for e in errors) + assert not any("file_type" in e for e in errors) def test_validate_semantic_fragment_accepts_rationale_file_type(): diff --git a/tests/test_semantic_id_remap_root.py b/tests/test_semantic_id_remap_root.py index ab85bd6d7..3a5fee626 100644 --- a/tests/test_semantic_id_remap_root.py +++ b/tests/test_semantic_id_remap_root.py @@ -48,3 +48,36 @@ def test_normal_semantic_remap_still_works(): remap = _semantic_id_remap( [{"id": "foo", "source_file": "src/foo.py", "_origin": "semantic"}], "/proj") assert isinstance(remap, dict) + + +# --- #1917: _semantic_id_remap must be idempotent (no id accretion) --- + +def test_semantic_id_remap_is_idempotent_when_stem_contains_legacy_stem(): + """A file whose parent dir name equals its stem (.claude/CLAUDE.md -> + canonical `claude_claude`, legacy `claude`) must not re-prefix an + already-canonical id on every build (#1917). Without the guard, ids grow + `claude_x` -> `claude_claude_x` -> `claude_claude_claude_x` ..., defeating + the same_topology/no_change short-circuits.""" + nodes = [{"id": "claude_graphify_trigger", + "source_file": ".claude/CLAUDE.md", "_origin": "semantic"}] + first = _semantic_id_remap(nodes, ".") + assert first == {"claude_graphify_trigger": "claude_claude_graphify_trigger"} + # Feed the migrated ids back through: a second pass must be a fixed point. + migrated = [{**n, "id": first.get(n["id"], n["id"])} for n in nodes] + assert _semantic_id_remap(migrated, ".") == {}, "id re-prefixed on second build (#1917)" + + +def test_semantic_id_remap_bare_file_node_is_idempotent(): + """The bare file node id follows the same fixed-point rule.""" + nodes = [{"id": "claude", "source_file": ".claude/CLAUDE.md", "_origin": "semantic"}] + first = _semantic_id_remap(nodes, ".") + assert first == {"claude": "claude_claude"} + migrated = [{"id": "claude_claude", "source_file": ".claude/CLAUDE.md", "_origin": "semantic"}] + assert _semantic_id_remap(migrated, ".") == {} + + +def test_semantic_id_remap_still_migrates_genuine_legacy_id(): + """The idempotency guard must not block a real one-time legacy migration: + a pre-scheme id under a normal path still remaps once to the canonical stem.""" + nodes = [{"id": "readme_booking", "source_file": "api/README.md", "_origin": "semantic"}] + assert _semantic_id_remap(nodes, ".") == {"readme_booking": "api_readme_booking"} diff --git a/tests/test_serve.py b/tests/test_serve.py index 2647aa1a8..fdaa2753c 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -7,7 +7,10 @@ from graphify.serve import ( _communities_from_graph, _score_nodes, + _score_query, _compute_idf, + _EXACT_MATCH_BONUS, + _SOURCE_MATCH_BONUS, _pick_seeds, _bfs, _dfs, @@ -22,8 +25,10 @@ _query_graph_text, _resolve_context_filters, _subgraph_to_text, + _cut_lines_to_budget, _load_graph, _community_header, + _search_tokens, ) @@ -123,6 +128,66 @@ def _add(nid, label, src): assert scored[0][0] > scored[1][0], "exact label must strictly outrank superset/token-bag matches" +def test_score_nodes_coverage_lone_generic_exact_hit_loses_to_multi_term_match(): + """A lone generic-word exact match must not bury a multi-term match. + + Reproduces #1602: in a multi-term query, a single generic term that + exactly equals a short leaf label (query term "list" vs a list() function + node) received the full exact-tier bonus and outranked every node matching + several of the query's terms, even when the query contained the target's + literal identifier. The per-term exact/prefix tiers are now scaled by + squared term coverage, so a 1-of-5-terms collision drops below a + multi-term match. The leaves live in the same directory as the target + (the realistic case) to pin that source-path hits do not count as + coverage and hand the collision its exact tier back. + """ + G = nx.Graph() + + def _add(nid, label, src): + G.add_node(nid, label=label, norm_label=label.lower(), + source_file=src, community=0) + + _add("target", "ClientLive.Index", "lib/clients_live/index.ex") + _add("form", "ClientLive.Form", "lib/clients_live/form.ex") + _add("show", "ClientLive.Show", "lib/clients_live/show.ex") + # Same-named tiny leaf functions: "list" == bare label fires the exact + # tier. Placed in the target's own directory so their source paths also + # substring-match the query term "clients": a path hit must not inflate + # the coverage that multiplies the exact tier. + for i in range(3): + _add(f"leaf{i}", "list()", f"lib/clients_live/helpers{i}.ex") + # Filler making "list" a common (low-IDF) token, as in a real graph where + # list()/get()/new() style names are ubiquitous. + for i in range(24): + _add(f"filler{i}", f"shopping list {i}", f"lib/filler{i}.ex") + + # The user pastes the real identifier plus context words; tokenization + # yields 5 terms: clientlive, index, clients, list, columns. + scored = _score_nodes(G, [t.lower() for t in "ClientLive.Index clients list columns".split()]) + by_id = {nid: s for s, nid in scored} + + assert scored[0][1] == "target" + assert by_id["target"] > by_id["leaf0"], ( + "a 1-of-5-terms exact collision must not outrank the node matching 3 of 5 terms" + ) + + +def test_score_nodes_coverage_full_coverage_query_is_unchanged(): + """Coverage scaling must not touch full-coverage queries (coverage == 1). + + A single-term identifier lookup keeps the exact tier's full magnitude, so + `query "FooBarService"` behavior is byte-identical to before #1602. + """ + G = _make_graph() + scored = _score_nodes(G, ["extract"]) + w = _compute_idf(G, ["extract"])["extract"] + assert scored[0][1] == "n1" + # Full-query exact tier (10x) + per-term exact tier + source hit + # ("extract" in "extract.py"), all undampened. + expected = (_EXACT_MATCH_BONUS * 10 + _EXACT_MATCH_BONUS + _SOURCE_MATCH_BONUS) * w + assert scored[0][0] == pytest.approx(expected) + + def test_find_node_ignores_trailing_punctuation(): G = _make_graph() assert _find_node(G, "extract?") == ["n1"] @@ -135,6 +200,30 @@ def test_find_node_matches_full_punctuated_unicode_label(): assert _find_node(G, "Skill /auditar — Auditoría inquisitiva de enlaces") == ["n1"] +def test_find_node_matches_punctuated_file_label_exactly(): + # #1704: an exactly-typed punctuated file label must resolve through explain, + # just like it does through path/query. + G = nx.Graph() + G.add_node("f1", label="blockStream.ts", norm_label="blockstream.ts", + source_file="lib/blockStream.ts", source_location="L1") + G.add_node("f2", label="blockStream.test.ts", norm_label="blockstream.test.ts", + source_file="lib/blockStream.test.ts", source_location="L1") + assert _find_node(G, "blockStream.ts")[0] == "f1" + assert _find_node(G, "blockStream.test.ts")[0] == "f2" + + +def test_find_node_resolves_when_label_and_norm_label_diverge(): + # #1704 hardening: the tokenized-label tier only rescues the match by + # coincidence (label tokenizes the same as the query). When `label` and + # `norm_label` diverge, only the symmetric `norm_query == norm_label` match + # resolves it. Here label tokenizes to "blockstream" but norm_label is + # "blockstream.ts" — this fails without the norm_query path. + G = nx.Graph() + G.add_node("n1", label="BlockStream", norm_label="blockstream.ts", + source_file="lib/x.ts", source_location="L1") + assert _find_node(G, "blockStream.ts") == ["n1"] + + # --- trigram candidate prefilter (the trigram index that shrinks the O(N) scan) --- @@ -278,6 +367,43 @@ def test_query_terms_all_stopwords_falls_back_to_unfiltered(): assert _query_terms("how does it work") == ["how", "does", "work"] +def test_query_terms_drops_german_question_stopwords(): + # #1900: German full-sentence queries must reduce to the content noun. + # In a mostly-English corpus "wie"/"funktioniert" are rare, get high IDF + # weight, and out-seed the actual keyword unless dropped here. + assert _query_terms("Wie funktioniert die Authentifizierung?") == ["authentifizierung"] + + +def test_query_terms_all_german_stopwords_falls_back_to_unfiltered(): + # Existing all-stopword fallback applies to German fillers too: the query + # keeps its terms rather than seeding on nothing. + terms = _query_terms("wie funktioniert das") + assert terms == ["wie", "funktioniert", "das"] + + +def test_pick_seeds_german_query_seeds_content_node_not_heading_noise(): + """End-to-end for #1900: a German question over a graph with German + heading-noise nodes must seed on the content noun, not on nodes that + happen to contain 'die'/'wie'/'wird'.""" + G = nx.DiGraph() + G.add_node("cfg", label="Die Konfiguration", source_file="docs/konfiguration.md") + G.add_node("sec", label="Wie wird gesichert", source_file="docs/sicherheit.md") + G.add_node("auth", label="Authentifizierung", source_file="src/auth.py") + G.add_node("helper", label="login_helper", source_file="src/auth.py") + G.add_edge("helper", "auth") + + q = "Wie funktioniert die Authentifizierung?" + terms = _query_terms(q) + # #1918: _score_query does combined scoring + per-term singleton winners in + # one traversal; _pick_seeds consumes best_seed_by_term for the per-term + # guarantee (replaces the old terms= per-term rescoring). + qs = _score_query(G, terms, collect_per_term_seeds=True) + seeds = _pick_seeds(qs.ranked, G=G, best_seed_by_term=qs.best_seed_by_term) + assert "auth" in seeds + assert "cfg" not in seeds + assert "sec" not in seeds + + def test_query_terms_filters_only_short_english_terms(monkeypatch): import graphify.serve as serve_mod @@ -474,6 +600,32 @@ def test_load_graph_missing_file(tmp_path): _load_graph(str(graphify_dir / "nonexistent.json")) +def test_load_graph_corrupted_json_prints_recovery_message(tmp_path, capsys): + """json.JSONDecodeError is a ValueError subclass, so its except clause + must be checked before the bare (ValueError, FileNotFoundError) clause, + or the corrupted-graph recovery hint is unreachable (#2005).""" + p = tmp_path / "graph.json" + p.write_text("{not valid json") + with pytest.raises(SystemExit): + _load_graph(str(p)) + err = capsys.readouterr().err + assert "graph.json is corrupted" in err + assert "Re-run /graphify to rebuild" in err + + +def test_load_graph_generic_value_error_message_unchanged(tmp_path, capsys): + """A non-decode ValueError (e.g. a non-.json path) must still print the + generic error, not the corrupted-graph hint — pins the except-clause + order from #2005 so a future refactor can't collapse them back.""" + p = tmp_path / "graph.txt" + p.write_text("not a graph") + with pytest.raises(SystemExit): + _load_graph(str(p)) + err = capsys.readouterr().err + assert "must be a .json file" in err + assert "corrupted" not in err + + def test_load_graph_rejects_oversized_file(monkeypatch, tmp_path, capsys): # #F4: oversized graph.json must fail fast (SystemExit) with a clear error. G = _make_graph() @@ -648,8 +800,8 @@ def test_pick_seeds_respects_max_k(): def test_pick_seeds_without_diversity_args_is_unchanged(): - """G/terms are optional and default to None: existing callers see identical - behavior to before this change.""" + """G/best_seed_by_term are optional and default to None: existing callers + see identical behavior to before this change.""" scored = [(1000.0, "fbs"), (1.0, "err1"), (0.9, "err2")] assert _pick_seeds(scored) == ["fbs"] @@ -658,9 +810,9 @@ def test_pick_seeds_diversity_recovers_starved_term(monkeypatch): """Reproduces #1445: a vague natural-language query where one term's incidental EXACT match on an unrelated node (e.g. a common word also used as an unrelated field/identifier) outscores every SUBSTRING match on the - query's other, actually-relevant terms by ~1000x. Without G/terms, the - 20%-gap cutoff discards the relevant candidate entirely; with them, it is - recovered as a guaranteed per-term seed. + query's other, actually-relevant terms by ~1000x. Without + G/best_seed_by_term, the 20%-gap cutoff discards the relevant candidate + entirely; with them, it is recovered as a guaranteed per-term seed. """ G = nx.DiGraph() # "unrelated" is an exact label match for the query term "unrelated" and @@ -672,17 +824,81 @@ def test_pick_seeds_diversity_recovers_starved_term(monkeypatch): G.add_edge("other", "target") terms = ["unrelated", "widget"] - scored = _score_nodes(G, terms) + # `_score_query` does the combined scoring and the per-term singleton + # winner tracking in one traversal; `_pick_seeds` consumes its + # `best_seed_by_term` to satisfy the per-term guarantee without rescoring. + qs = _score_query(G, terms, collect_per_term_seeds=True) + scored = qs.ranked # Sanity check the premise: without diversity, only the exact match survives. seeds_before = _pick_seeds(scored) assert seeds_before == ["noise"] - seeds_after = _pick_seeds(scored, G=G, terms=terms) + seeds_after = _pick_seeds(scored, G=G, best_seed_by_term=qs.best_seed_by_term) assert "noise" in seeds_after assert "target" in seeds_after +# --- generic-symbol seed flooding (#1766) --- + +def test_pick_seeds_dedups_homonymous_generic_labels(): + """Many nodes sharing one generic label (e.g. framework `GET` handlers) + must contribute at most ONE seed, not consume every slot (#1766). A + distinct, relevant label still gets its own seed.""" + G = nx.DiGraph() + for i in range(5): + G.add_node(f"get{i}", label="GET", source_file=f"routes/r{i}.py") + G.add_node("um", label="users_model", source_file="models/users.py") + # Score all the GET nodes above users_model so, pre-fix, they'd take every slot. + scored = [(1000.0, f"get{i}") for i in range(5)] + [(900.0, "um")] + seeds = _pick_seeds(scored, G=G) + get_seeds = [s for s in seeds if s.startswith("get")] + assert len(get_seeds) == 1, f"expected one GET representative, got {get_seeds}" + # A different, well-within-gap label is not starved out by the GET flood. + assert "um" in seeds + + +def test_pick_seeds_dedup_key_is_case_and_diacritic_normalized(): + """`GET`/`Get`/`get` are the same generic label and must dedup together.""" + G = nx.DiGraph() + G.add_node("a", label="GET", source_file="a.py") + G.add_node("b", label="Get", source_file="b.py") + G.add_node("c", label="get", source_file="c.py") + scored = [(1000.0, "a"), (990.0, "b"), (980.0, "c")] + seeds = _pick_seeds(scored, G=G) + assert len(seeds) == 1, f"case-variant duplicates not collapsed: {seeds}" + + +def test_pick_seeds_per_term_guarantee_does_not_reintroduce_generic_dupe(monkeypatch): + """The per-term guarantee loop must honor the same per-label cap, so it can't + add a second `GET` after dedup already seeded one (#1766).""" + G = nx.DiGraph() + for i in range(3): + G.add_node(f"get{i}", label="GET", source_file=f"r{i}.py") + G.add_node("um", label="users_model", source_file="users.py") + G.add_edge("um", "get0") + terms = ["get", "users"] + qs = _score_query(G, terms, collect_per_term_seeds=True) + seeds = _pick_seeds(qs.ranked, G=G, best_seed_by_term=qs.best_seed_by_term) + get_seeds = [s for s in seeds if s.startswith("get")] + assert len(get_seeds) == 1, f"per-term guarantee reintroduced a GET dupe: {seeds}" + + +def test_score_nodes_scores_identical_labels_equally(): + """Guard against a per-label multiplicity penalty leaking into _score_nodes + (shared by shortest_path / explain endpoint resolution): two nodes with the + SAME label must receive the SAME score for a query, i.e. the fix lives in + seed selection, not in the shared scorer (#1766 followup).""" + G = nx.DiGraph() + G.add_node("g1", label="GET", source_file="a.py") + G.add_node("g2", label="GET", source_file="b.py") + G.add_node("g3", label="GET", source_file="c.py") + by_id = {nid: s for s, nid in _score_nodes(G, ["get"])} + assert by_id["g1"] == by_id["g2"] == by_id["g3"], ( + f"identical-label nodes scored differently: {by_id}" + ) + + # --- actionable truncation hint (#897) --- def test_subgraph_to_text_truncation_hint_is_actionable(): @@ -829,3 +1045,344 @@ def test_community_header_sanitizes_name(): out = _community_header(3, "Pay\x00ments\x1b[31m") assert out.startswith("Community 3 — ") assert "\x00" not in out and "\x1b" not in out + + +# --- single-pass scoring refactor: reference-impl equality + one-traversal --- + + +def _reference_best_seed_by_term(G: nx.Graph, terms: list[str]) -> dict[str, str]: + """Test-only oracle for the legacy per-term `_pick_seeds(terms=...)` loop. + + Re-creates what `_pick_seeds` did before the single-pass refactor: rescore + the whole graph per token via `_score_nodes(G, [token])`, take the top- + scoring ties, and break them by `max(tied, key=degree)` (which, over a + list sorted by `(-score, label_len, nid)`, returns the highest-degree node + with ties broken toward the shortest label then the smallest node id). + This is the semantics `_score_query(..., collect_per_term_seeds=True)` now + produces inline during its single traversal. + """ + norm_terms = sorted({tok for t in terms for tok in _search_tokens(t)}) + best: dict[str, str] = {} + for term in norm_terms: + term_scored = _score_nodes(G, [term]) + if not term_scored: + continue + best_score = term_scored[0][0] + tied = [nid for s, nid in term_scored if s == best_score] + best_nid = max(tied, key=lambda n: G.degree(n)) if len(tied) > 1 else term_scored[0][1] + best[term] = best_nid + return best + + +def _make_random_scoring_graph(n: int, *, seed: int) -> nx.DiGraph: + """Reproducible broad-match DiGraph: short constructed labels + edge noise. + + Labels draw from a small syllable pool so tokens collide across nodes, + forcing the trigram prefilter to be selective and exercising score ties + on common tokens. Edge noise provides degree variance so the legacy + tie-break (`max(tied, key=degree)`) is exercised against the new + `(-singleton, -degree, label_len, nid)` key tuple. + """ + import random + + rng = random.Random(seed) + syllables = [ + "foo", "bar", "baz", "get", "set", "run", "user", "name", "path", + "build", "report", "extract", "router", "config", "service", + "handler", "token", "auth", "rate", "limit", "widget", "model", + ] + G: nx.DiGraph = nx.DiGraph() + for i in range(n): + label = "_".join(rng.sample(syllables, rng.randint(1, 3))) + G.add_node(f"n{i}", label=label, source_file=f"src/{label[:8]}.py") + for _ in range(n * 2): + a, b = rng.randrange(n), rng.randrange(n) + if a != b: + G.add_edge(f"n{a}", f"n{b}", relation="calls", confidence="EXTRACTED") + return G + + +SYLLABLE_QUERIES = [ + ["get"], # single token, exact-match + ["get", "user"], # two distinct tokens + ["router", "service", "handler"], # multi-token identifier + ["extract", "build", "report", "path"], # broad term + ["nonexistent"], # no matches + ["nonexistent", "get"], # one missing term + match + ["bar", "bar"], # repeated token (must dedupe) + ["baz", "run", "set", "auth", "rate", "limit"], # many tokens +] + + +@pytest.mark.parametrize("terms", SYLLABLE_QUERIES) +def test_score_query_ranked_matches_score_nodes_byte_identical(terms): + """`_score_query(..., collect_per_term_seeds=False).ranked` is the byte-for- + byte match of `_score_nodes(G, terms)` — guaranteeing path/explain/tests see + no behavior change from the refactor.""" + G = _make_random_scoring_graph(80, seed=7) + assert _score_query(G, terms, collect_per_term_seeds=False).ranked == _score_nodes(G, terms) + + +@pytest.mark.parametrize("terms", SYLLABLE_QUERIES) +def test_score_query_best_seed_by_term_matches_legacy_singleton_scoring(terms): + """Per-token winner the single-pass scorer records matches the legacy + `_score_nodes([token])` + `max(tied, key=degree)` oracle exactly.""" + G = _make_random_scoring_graph(80, seed=7) + ref = _reference_best_seed_by_term(G, terms) + opt = _score_query(G, terms, collect_per_term_seeds=True).best_seed_by_term + assert ref == opt, f"terms={terms}: legacy={ref} optimized={opt}" + + +@pytest.mark.parametrize("terms", SYLLABLE_QUERIES) +def test_pick_seeds_with_optimized_best_seed_matches_legacy_semantics(terms): + """The seeds produced by `_pick_seeds(qs.ranked, G=G, best_seed_by_term= + qs.best_seed_by_term)` exactly match what the legacy `_pick_seeds(terms=...)` + loop would have produced (recreated via the reference oracle).""" + G = _make_random_scoring_graph(80, seed=7) + qs = _score_query(G, terms, collect_per_term_seeds=True) + ref_best = _reference_best_seed_by_term(G, terms) + # Legacy `_pick_seeds(terms=...)` ran `_score_nodes(G, [term])` per token + # to build ref_best, then deduped by label key. The new `_pick_seeds( + # best_seed_by_term=...)` only swaps the source of the per-token winners, + # so it must produce the same seeds given equivalent inputs. + opt_seeds = _pick_seeds(qs.ranked, G=G, best_seed_by_term=qs.best_seed_by_term) + ref_seeds = _pick_seeds(qs.ranked, G=G, best_seed_by_term=ref_best) + assert opt_seeds == ref_seeds, f"terms={terms}: ref={ref_seeds} opt={opt_seeds}" + # Per-term guarantee: every legacy winner with a non-empty seed slot is + # accounted for — either it appears in the seed list or another node with + # the same normalized label already claimed the slot (#1766 label dedup). + ref_seed_set = set(ref_seeds) + for term, nid in ref_best.items(): + if nid in ref_seed_set: + continue + nid_label = (G.nodes[nid].get("norm_label") + or G.nodes[nid].get("label") + or nid) + seeded_with_same_label = any( + (G.nodes[s].get("norm_label") or G.nodes[s].get("label") or s) == nid_label + for s in ref_seeds + ) + assert seeded_with_same_label, ( + f"term {term!r} winner {nid!r} dropped without label-dedup reason" + ) + + +def test_score_query_matches_legacy_across_random_deterministic_graphs(): + """Across many deterministic random graphs and many random multi-term + queries, the single-pass scorer's combined ranking, per-token winners, + and resulting seed list all match the legacy semantics. Exercises label + collisions, ties, broad terms, missing terms, and graph size variance.""" + import random + + rng = random.Random(42) + syllables = [ + "foo", "bar", "baz", "get", "set", "run", "user", "name", "path", + "build", "report", "extract", "router", "config", "service", + "handler", "token", "auth", "rate", "limit", "widget", "model", + ] + for trial in range(30): + n = rng.randint(20, 200) + G = _make_random_scoring_graph(n, seed=rng.randint(0, 10**9)) + nq = rng.randint(1, 5) + terms = [rng.choice(syllables) for _ in range(nq)] + ref_best = _reference_best_seed_by_term(G, terms) + opt = _score_query(G, terms, collect_per_term_seeds=True) + # (a) Combined ranking unchanged. + assert opt.ranked == _score_nodes(G, terms), ( + f"trial {trial}: combined ranking diverged for terms={terms}" + ) + # (b) Per-token winners match the legacy per-term rescoring loop. + assert opt.best_seed_by_term == ref_best, ( + f"trial {trial}: best_seed_by_term diverged; ref={ref_best} opt={opt.best_seed_by_term}" + ) + # (c) Final seed list is identical under the legacy semantics. + ref_seeds = _pick_seeds(opt.ranked, G=G, best_seed_by_term=ref_best) + opt_seeds = _pick_seeds(opt.ranked, G=G, best_seed_by_term=opt.best_seed_by_term) + assert opt_seeds == ref_seeds, ( + f"trial {trial}: seeds diverged; ref={ref_seeds} opt={opt_seeds}" + ) + + +def test_score_query_matches_legacy_under_full_scan_fallback(monkeypatch): + """When the trigram prefilter falls back to a full-graph scan, the + single-pass path still produces identical rankings and per-term winners. + + Forces `_trigram_candidates` to return None so the combined iterates the + whole graph — mirroring per-token `_score_nodes([token])` which would also + full-scan when its own trigram search isn't selective.""" + monkeypatch.setattr( + "graphify.serve._trigram_candidates", lambda G, needles: None + ) + terms = ["router", "service", "handler"] + G = _make_random_scoring_graph(80, seed=19) + ref_best = _reference_best_seed_by_term(G, terms) + opt = _score_query(G, terms, collect_per_term_seeds=True) + assert opt.ranked == _score_nodes(G, terms) + assert opt.best_seed_by_term == ref_best + + +def test_query_graph_text_makes_exactly_one_score_query_call(monkeypatch): + """`_query_graph_text` must invoke `_score_query` exactly once per query, + regardless of how many tokens the query has — eliminating the legacy + T+1-pass rescoring. `_score_nodes` must NOT be called from the query path + (only path/explain still call it).""" + G = _make_random_scoring_graph(60, seed=23) + original_sq = _score_query + original_sn = _score_nodes + + state = {"sq": 0, "sn": 0} + + def counting_sq(*a, **k): + state["sq"] += 1 + return original_sq(*a, **k) + + def counting_sn(*a, **k): + state["sn"] += 1 + return original_sn(*a, **k) + + monkeypatch.setattr("graphify.serve._score_query", counting_sq) + monkeypatch.setattr("graphify.serve._score_nodes", counting_sn) + + queries = [ + "foo", # one term + "foo bar", # two + "router service handler", # three (the scenario the RFC targets) + "get user run name path", # five + "extract build report router config service token rate limit widget", # ten + ] + for q in queries: + state["sq"] = 0 + state["sn"] = 0 + _query_graph_text(G, q, mode="bfs", depth=1) + assert state["sq"] == 1, ( + f"expected exactly one _score_query call for {q!r}, got {state['sq']}" + ) + assert state["sn"] == 0, ( + f"query path must not call _score_nodes; got {state['sn']} call(s) for {q!r}" + ) + + +def test_score_query_collect_per_term_seeds_false_omits_tracking(monkeypatch): + """`collect_per_term_seeds=False` returns empty `best_seed_by_term` and + does not pay for per-token best tracking — preserving the cost contract + for path/explain/tests callers that only want the combined ranking.""" + G = _make_random_scoring_graph(50, seed=29) + qs = _score_query(G, ["foo", "bar", "baz"], collect_per_term_seeds=False) + assert qs.best_seed_by_term == {} + # And the combined output is still byte-identical to _score_nodes. + assert qs.ranked == _score_nodes(G, ["foo", "bar", "baz"]) + + +# --- BUG2: seed survival, truncation notice, deterministic ordering ---------- + +def _star_graph(n_spokes=40): + """A high-degree hub plus a low-degree answer node, to force the answer past + a pure degree-sorted / BFS cut unless seed-first ordering protects it.""" + G = nx.Graph() + G.add_node("hub", label="Hub", source_file="hub.py", source_location="L1", community=0) + for i in range(n_spokes): + G.add_node(f"s{i}", label=f"spoke{i}", source_file=f"s{i}.py", source_location="L1", community=0) + G.add_edge("hub", f"s{i}", relation="calls", confidence="EXTRACTED") + # low-degree answer node, attached to one spoke + G.add_node("answer", label="CompanySpacingGate", source_file="gate.py", + source_location="L12", community=0) + G.add_edge("s0", "answer", relation="calls", confidence="EXTRACTED") + return G + + +def test_subgraph_to_text_seed_survives_truncation(): + """BUG2: a low-degree answer node passed as a seed is rendered first and + survives a tiny budget, and truncation is announced.""" + G = _star_graph() + nodes = set(G.nodes) + text = _subgraph_to_text(G, nodes, list(G.edges()), token_budget=30, seeds=["answer"]) + assert "CompanySpacingGate" in text, "seed node was cut (BUG2)" + node_lines = [l for l in text.splitlines() if l.startswith("NODE ")] + assert "CompanySpacingGate" in node_lines[0], "seed must render first" + assert "TRUNCATED" in text + + +def test_query_graph_text_passes_seeds_so_answer_survives(): + """BUG2 regression guard: the query path must pass seeds to the renderer (a + branch merge had dropped the argument), so a queried low-degree symbol + appears in the body even when the output is truncated.""" + G = _star_graph() + text = _query_graph_text(G, "CompanySpacingGate", mode="bfs", depth=2, token_budget=40) + # Present in the body, not merely the Start: header. + body = text.split("\n\n", 1)[-1] + assert "CompanySpacingGate" in body + + +def test_subgraph_to_text_truncation_notice_at_top(): + G = _star_graph() + text = _subgraph_to_text(G, set(G.nodes), list(G.edges()), token_budget=30, seeds=["answer"]) + assert text.startswith("[!] TRUNCATED"), f"notice not at top: {text[:60]!r}" + assert "of" in text.splitlines()[0] and "nodes" in text.splitlines()[0] + assert "truncated" in text # end marker still present + + +def test_subgraph_to_text_no_notice_when_under_budget(): + G = _make_graph() + text = _subgraph_to_text(G, {"n1", "n2"}, [("n1", "n2")], token_budget=2000) + assert "TRUNCATED" not in text and "truncated" not in text + + +def test_subgraph_to_text_order_is_deterministic(): + """Equal-degree nodes render in a stable order regardless of set iteration.""" + G = nx.Graph() + for i in range(10): + G.add_node(f"z{i}", label=f"z{i}", source_file=f"z{i}.py", source_location="L1", community=0) + nodes = set(G.nodes) + a = _subgraph_to_text(G, nodes, []) + b = _subgraph_to_text(G, set(reversed(list(nodes))), []) + assert a == b + + +# --- #2069: token budget on get_neighbors / get_community line lists ---------- + +def test_cut_lines_to_budget_under_budget_is_byte_identical(): + lines = ["Neighbors of X:", " --> a [calls] [EXTRACTED]", " --> b [calls] [EXTRACTED]"] + out = _cut_lines_to_budget(lines, token_budget=2000, narrow_hint="use relation_filter") + assert out == "\n".join(lines) + assert "TRUNCATED" not in out and "truncated" not in out + + +def test_cut_lines_to_budget_over_budget_announces_at_top(): + lines = [f" --> node{i} [calls] [EXTRACTED]" for i in range(200)] + out = _cut_lines_to_budget(lines, token_budget=20, narrow_hint="use get_node for a specific symbol") + # Top notice (silence must not read as absence) + accurate counts + bottom marker + hint. + assert out.startswith("[!] TRUNCATED: showing ") + first = out.splitlines()[0] + assert "of 200 lines" in first + assert "use get_node for a specific symbol" in out + assert "truncated" in out # end marker retained + # shown count in the notice matches the actual kept line count. + import re + shown = int(re.search(r"showing (\d+) of", first).group(1)) + body = out.split("\n\n", 1)[1].split("\n... (truncated", 1)[0] + assert body.count("\n") + 1 == shown + + +def test_subgraph_to_text_ignores_dangling_src_tgt(monkeypatch): + """#2080 review: a stray/dangling _src/_tgt on an edge (hand-edited or + adversarial graph.json) must NOT crash rendering; fall back to (u, v).""" + G = nx.Graph() + G.add_node("a", label="Alpha", source_file="a.py", source_location="L1", community=0) + G.add_node("b", label="Beta", source_file="b.py", source_location="L2", community=0) + # _src names a node that doesn't exist -> must be ignored, no KeyError. + G.add_edge("a", "b", relation="calls", confidence="EXTRACTED", _src="ghost", _tgt="b") + out = _subgraph_to_text(G, {"a", "b"}, [("a", "b")]) + assert "EDGE" in out and "Alpha" in out and "Beta" in out # rendered, didn't crash + + +def test_subgraph_to_text_honors_valid_src_tgt_direction(): + """#2080: a valid _src/_tgt (the stored direction) is honored even when the + traversal tuple is reversed.""" + G = nx.Graph() + G.add_node("caller", label="caller", source_file="c.py", source_location="L1", community=0) + G.add_node("callee", label="callee", source_file="d.py", source_location="L2", community=0) + # Edge collected as (callee, caller) by traversal, but stored direction is caller->callee. + G.add_edge("callee", "caller", relation="calls", confidence="EXTRACTED", _src="caller", _tgt="callee") + out = _subgraph_to_text(G, {"caller", "callee"}, [("callee", "caller")]) + edge_line = next(l for l in out.splitlines() if l.startswith("EDGE")) + assert "caller --calls" in edge_line and "--> callee" in edge_line diff --git a/tests/test_settings_merge.py b/tests/test_settings_merge.py new file mode 100644 index 000000000..ab4cd562b --- /dev/null +++ b/tests/test_settings_merge.py @@ -0,0 +1,204 @@ +"""Regression tests for issue #2167: hook installers must merge into existing +settings/hooks JSON files, never clobber them. + +The old behavior fell back to ``settings = {}`` on any parse error (a UTF-8 BOM +was enough, same class as #2163) and then rewrote the whole file, destroying the +user's mcpServers/enabledPlugins/theme/hooks. The fix: + +- read with utf-8-sig (BOM-tolerant), +- refuse to touch an existing file that is not a JSON object (stderr + exit 1), +- back up to .graphify-bak before any modifying write, +- skip the write entirely when nothing changed (idempotent re-install), +- never crash on (and always preserve) non-dict hook entries. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from graphify.install import ( + _install_claude_hook, + _install_codebuddy_hook, + _install_codex_hook, + _install_gemini_hook, +) + +# installer key -> (function, settings file relative to project dir, hooks section) +_INSTALLERS = { + "claude": (_install_claude_hook, Path(".claude") / "settings.json", "PreToolUse"), + "codebuddy": (_install_codebuddy_hook, Path(".codebuddy") / "settings.json", "PreToolUse"), + "codex": (_install_codex_hook, Path(".codex") / "hooks.json", "PreToolUse"), + "gemini": (_install_gemini_hook, Path(".gemini") / "settings.json", "BeforeTool"), +} + +ALL_INSTALLERS = pytest.mark.parametrize("installer", sorted(_INSTALLERS), ids=sorted(_INSTALLERS)) + + +def _seed(tmp_path: Path, installer: str, payload) -> Path: + _, rel, _ = _INSTALLERS[installer] + settings_path = tmp_path / rel + settings_path.parent.mkdir(parents=True, exist_ok=True) + if isinstance(payload, bytes): + settings_path.write_bytes(payload) + else: + settings_path.write_text(json.dumps(payload, indent=2), encoding="utf-8") + return settings_path + + +def _run(tmp_path: Path, installer: str, **kwargs) -> Path: + fn, rel, _ = _INSTALLERS[installer] + fn(tmp_path, **kwargs) + return tmp_path / rel + + +# ---------------------------------------------------------------- merge + + +def test_claude_install_preserves_existing_settings(tmp_path): + """#2167 core case: every key graphify does not own must survive install.""" + seeded = { + "mcpServers": {"context7": {"command": "npx", "args": ["context7"]}}, + "enabledPlugins": ["my-plugin@marketplace"], + "theme": "dark", + "hooks": { + "PostToolUse": [ + {"matcher": "Bash", "hooks": [{"type": "command", "command": "my-formatter"}]} + ], + "PreToolUse": [ + {"matcher": "Write", "hooks": [{"type": "command", "command": "my-write-guard"}]} + ], + }, + } + settings_path = _seed(tmp_path, "claude", seeded) + + _run(tmp_path, "claude", strict=True) + + result = json.loads(settings_path.read_text(encoding="utf-8")) + # top-level keys graphify does not own are untouched + assert result["mcpServers"] == seeded["mcpServers"] + assert result["enabledPlugins"] == seeded["enabledPlugins"] + assert result["theme"] == "dark" + # hooks sections graphify does not manage are untouched + assert result["hooks"]["PostToolUse"] == seeded["hooks"]["PostToolUse"] + # the user's own PreToolUse entry survives alongside graphify's + pre_tool = result["hooks"]["PreToolUse"] + assert seeded["hooks"]["PreToolUse"][0] in pre_tool + graphify_hooks = [h for h in pre_tool if "graphify" in str(h)] + assert len(graphify_hooks) == 2 + # strict=True lands on the read guard + assert any(h["hooks"][0]["command"].endswith("--strict") for h in graphify_hooks) + + +# ---------------------------------------------------------------- BOM + + +@ALL_INSTALLERS +def test_bom_settings_are_merged_not_clobbered(tmp_path, installer): + """A UTF-8 BOM must not trigger the parse-error path that used to clobber.""" + seeded = {"mcpServers": {"keep": {"command": "keep-me"}}, "theme": "dark"} + body = json.dumps(seeded, indent=2).encode("utf-8") + settings_path = _seed(tmp_path, installer, b"\xef\xbb\xbf" + body) + + _run(tmp_path, installer) + + result = json.loads(settings_path.read_text(encoding="utf-8")) + assert result["mcpServers"] == seeded["mcpServers"] + assert result["theme"] == "dark" + section = _INSTALLERS[installer][2] + assert any("graphify" in str(h) for h in result["hooks"][section]) + + +# ---------------------------------------------------------------- invalid JSON + + +@ALL_INSTALLERS +def test_invalid_json_aborts_without_clobbering(tmp_path, installer, capsys): + """An unparseable existing file must abort the install, byte-identical on disk.""" + settings_path = _seed(tmp_path, installer, b"{ not json") + original = settings_path.read_bytes() + + with pytest.raises(SystemExit) as excinfo: + _run(tmp_path, installer) + + assert excinfo.value.code == 1 + assert str(settings_path) in capsys.readouterr().err + assert settings_path.read_bytes() == original + assert not settings_path.with_name(settings_path.name + ".graphify-bak").exists() + + +@ALL_INSTALLERS +def test_non_object_top_level_aborts_without_clobbering(tmp_path, installer, capsys): + """Valid JSON that is not an object (e.g. a list) must also refuse, not crash.""" + settings_path = _seed(tmp_path, installer, b'["not", "an", "object"]') + original = settings_path.read_bytes() + + with pytest.raises(SystemExit) as excinfo: + _run(tmp_path, installer) + + assert excinfo.value.code == 1 + assert str(settings_path) in capsys.readouterr().err + assert settings_path.read_bytes() == original + + +def test_non_dict_hooks_section_aborts(tmp_path, capsys): + """A malformed hooks value (not a dict) refuses instead of raising/clobbering.""" + settings_path = _seed(tmp_path, "claude", {"hooks": "oops", "theme": "dark"}) + original = settings_path.read_bytes() + + with pytest.raises(SystemExit) as excinfo: + _run(tmp_path, "claude") + + assert excinfo.value.code == 1 + assert str(settings_path) in capsys.readouterr().err + assert settings_path.read_bytes() == original + + +# ---------------------------------------------------------------- backup + + +@ALL_INSTALLERS +def test_backup_written_before_modify_and_stable_on_reinstall(tmp_path, installer): + seeded = {"theme": "dark", "mcpServers": {"keep": {}}} + settings_path = _seed(tmp_path, installer, seeded) + pre_write = settings_path.read_text(encoding="utf-8") + backup = settings_path.with_name(settings_path.name + ".graphify-bak") + + _run(tmp_path, installer) + + assert backup.exists() + assert backup.read_text(encoding="utf-8") == pre_write + merged = settings_path.read_text(encoding="utf-8") + assert merged != pre_write # sanity: the run was a modifying one + + # Idempotent second run: output is unchanged, so neither the settings file + # nor the backup may be rewritten (the backup keeps the pre-graphify content). + _run(tmp_path, installer) + assert settings_path.read_text(encoding="utf-8") == merged + assert backup.read_text(encoding="utf-8") == pre_write + + +def test_no_backup_on_fresh_install(tmp_path): + settings_path = _run(tmp_path, "claude") + assert settings_path.exists() + assert not settings_path.with_name(settings_path.name + ".graphify-bak").exists() + + +# ---------------------------------------------------------------- non-dict entries + + +@ALL_INSTALLERS +def test_non_dict_hook_entry_is_preserved_not_fatal(tmp_path, installer): + """A legacy non-dict entry in the managed section must not crash the filter + (the old claude/codebuddy filter called h.get() unconditionally) and must + survive the merge.""" + section = _INSTALLERS[installer][2] + settings_path = _seed(tmp_path, installer, {"hooks": {section: ["legacy-string"]}}) + + _run(tmp_path, installer) + + entries = json.loads(settings_path.read_text(encoding="utf-8"))["hooks"][section] + assert "legacy-string" in entries + assert any("graphify" in str(h) for h in entries) diff --git a/tests/test_skillgen.py b/tests/test_skillgen.py index 282aedfbf..0c09e601e 100644 --- a/tests/test_skillgen.py +++ b/tests/test_skillgen.py @@ -487,7 +487,8 @@ def test_monoliths_change_only_sanctioned_lines(): The round-trip (multiset diff vs the pinned v8 blob) must come back clean: each added/removed line matches one of the documented sanctioned predicates in gen — the enum unification, the unified description, the chunk-cleanup - rewrite (#1172), and the four #1392 runbook fixes. Anything else is drift. + rewrite (#1172), the four #1392 runbook fixes, and semantic-cache source + scoping (#1757). Anything else is drift. """ platforms = gen.load_platforms() for key in ("aider", "devin"): @@ -534,6 +535,16 @@ def test_monoliths_carry_the_1392_runbook_fixes(): assert "if not wrote:" in body +def test_monoliths_scope_semantic_cache_writes_to_uncached_files(): + """#1757: generated monoliths pass the dispatched-file allowlist when + replacing semantic cache entries.""" + platforms = gen.load_platforms() + for key in ("aider", "devin"): + body = gen.render(platforms[key])[0].content + assert ".graphify_uncached.txt').read_text(" in body + assert "allowed_source_files=uncached" in body + + def test_generated_runbooks_pass_root_to_save_manifest(): """#1417: every save_manifest call in a shipped runbook threads root=. @@ -938,3 +949,31 @@ def test_agents_audit_baseline_is_amps_v8_body(): assert gen._v8_baseline_ref("agents") == "47042beb05d1f6dd2186c0c499ae2840ce604ead:graphify/skill-amp.md" problems = gen.audit_coverage(platforms["agents"]) assert problems == [], "\n".join(problems) + + +def test_semantic_cache_calls_pass_prompt_file_for_every_split_host(): + """#1939: a skill's cache read and write must both name the extraction prompt + they use, or the run replays entries produced by an older prompt (the read) / + strands its results where the next read won't look (the write). + + Locked per host because the two calls live ~80 lines apart in the rendered + body: adding the argument to one and not the other silently disables the + cache rather than failing loudly. The monolith hosts (aider, devin) inline + their prompt instead of shipping references/extraction-spec.md and are + deliberately excluded — they have no spec path to point at. + """ + platforms = gen.load_platforms() + arts = gen.render_all(platforms) + bodies = [a for a in arts + if "check_semantic_cache(" in a.content + and "references/extraction-spec.md" in a.content] + assert bodies, "no rendered split-host skill body calls check_semantic_cache" + for a in bodies: + for call in ("check_semantic_cache(", "save_semantic_cache("): + line = next(ln for ln in a.content.splitlines() if call in ln and "import" not in ln) + assert "prompt_file='SPEC_PATH'" in line, ( + f"{a.path}: {call} must pass prompt_file so entries are attributed " + f"to the extraction prompt (#1939) — got: {line.strip()}" + ) + # The placeholder is inert unless the body tells the agent what to substitute. + assert "SPEC_PATH below is the **absolute** path" in a.content, a.path diff --git a/tests/test_src_layout_import_resolution.py b/tests/test_src_layout_import_resolution.py new file mode 100644 index 000000000..beeb6b95b --- /dev/null +++ b/tests/test_src_layout_import_resolution.py @@ -0,0 +1,153 @@ +"""#2072: Python import resolution must not depend on the scan root. + +A src-layout project (code under `src/`) used to lose most of its `imports` / +`imports_from` edges when scanned from the repo root, because absolute imports +were resolved only against the scan root while file-node ids are scan-root +relative. The same project scanned from `src/` resolved fine — so the chosen +scan root silently changed the graph. +""" +from __future__ import annotations + +from pathlib import Path + +from graphify.extract import extract +from graphify.extractors.resolution import _resolve_python_module_path +from graphify.build import build_from_json + + +_FILES = { + "mypkg/__init__.py": "from mypkg.core import Engine\n", + "mypkg/core.py": "class Engine:\n pass\n", + "mypkg/helpers.py": "def helper():\n return 1\n", + "mypkg/app.py": ( + "from mypkg.core import Engine\n" + "import mypkg.helpers\n\n" + "def run():\n return mypkg.helpers.helper()\n" + ), +} + + +def _write(base: Path, prefix: str = "") -> list[Path]: + written = [] + for rel, body in _FILES.items(): + p = base / prefix / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(body, encoding="utf-8") + written.append(p) + return written + + +def _import_edges(G): + """(relation, source, target) for import edges, present-endpoints only.""" + return { + (d.get("relation"), u, v) + for u, v, d in G.edges(data=True) + if d.get("relation") in ("imports", "imports_from") + } + + +def test_resolve_python_module_path_walks_up_to_src_package_root(tmp_path): + (tmp_path / "src" / "mypkg").mkdir(parents=True) + core = tmp_path / "src" / "mypkg" / "core.py" + core.write_text("class Engine: pass\n") + app = tmp_path / "src" / "mypkg" / "app.py" + app.write_text("from mypkg.core import Engine\n") + # scan root is the repo, code is under src/: must still resolve. + resolved = _resolve_python_module_path("mypkg.core", app, tmp_path, level=0) + assert resolved == core + # flat layout (package at root) is unchanged. + (tmp_path / "flat").mkdir() + (tmp_path / "flat" / "mod.py").write_text("x = 1\n") + assert _resolve_python_module_path("flat.mod", tmp_path / "flat" / "a.py", tmp_path, 0) == ( + tmp_path / "flat" / "mod.py" + ) + + +def test_import_edges_identical_from_root_or_src(tmp_path): + """Headline (#2072): the same project yields the same import edges whether + scanned from the repo root or from src/ (modulo the `src_` id prefix).""" + direct = tmp_path / "direct" + nested = tmp_path / "nested" + _write(direct) # direct/mypkg/... + _write(nested, prefix="src") # nested/src/mypkg/... (byte-identical) + + dpaths = [direct / r for r in _FILES] + npaths = [nested / "src" / r for r in _FILES] + dG = build_from_json(extract(dpaths, cache_root=tmp_path / "cd", root=direct, parallel=False), root=str(direct)) + nG = build_from_json(extract(npaths, cache_root=tmp_path / "cn", root=nested, parallel=False), root=str(nested)) + + d_edges = _import_edges(dG) + # strip the `src_` prefix the nested layout adds to every id. + n_edges = { + (rel, u[4:] if u.startswith("src_") else u, v[4:] if v.startswith("src_") else v) + for rel, u, v in _import_edges(nG) + } + assert d_edges, "sanity: the flat layout must produce import edges" + assert n_edges == d_edges, ( + f"scan root changed the import graph (#2072)\n root-only: {d_edges - n_edges}\n src-only: {n_edges - d_edges}" + ) + # Concretely, in the src layout: app<->core are connected by an import edge + # (endpoint order is storage-dependent on an undirected graph), and no import + # endpoint is a bare, unresolved `mypkg_*` id — every target resolved to a + # real `src_mypkg_*` file/symbol node. + n_imports = _import_edges(nG) + assert any({"src_mypkg_app"} <= {u, v} and any(n.startswith("src_mypkg_core") for n in (u, v)) + for _, u, v in n_imports), f"app->core import not resolved: {n_imports}" + endpoints = {n for _, u, v in n_imports for n in (u, v)} + assert not any(n.startswith("mypkg_") for n in endpoints), ( + f"unresolved bare import id survived (scan-root-relative mismatch): {endpoints}" + ) + + +def test_ambiguous_package_alias_is_not_repointed(tmp_path): + """A dotted-module id claimed by two different files (two src roots with the + same package) must stay dangling rather than pick an arbitrary file.""" + for sub in ("a", "b"): + d = tmp_path / sub / "src" / "pkg" + d.mkdir(parents=True) + (d / "__init__.py").write_text("") + (d / "mod.py").write_text("def f():\n return 1\n") + (tmp_path / "a" / "src" / "pkg" / "app.py").write_text("import pkg.mod\n") + paths = [ + tmp_path / "a" / "src" / "pkg" / "app.py", + tmp_path / "a" / "src" / "pkg" / "mod.py", + tmp_path / "b" / "src" / "pkg" / "mod.py", + tmp_path / "a" / "src" / "pkg" / "__init__.py", + tmp_path / "b" / "src" / "pkg" / "__init__.py", + ] + G = build_from_json(extract(paths, cache_root=tmp_path / "c", root=tmp_path, parallel=False), root=str(tmp_path)) + # The ambiguous `pkg_mod` alias claimed by both a/ and b/ must not be + # repointed onto either file — no fabricated cross-tree import edge. + imports = _import_edges(G) + targets = {v for _, _, v in imports} + # Neither file may be chosen — an ambiguous alias must stay dangling. + assert "a_src_pkg_mod" not in targets and "b_src_pkg_mod" not in targets, ( + f"ambiguous alias was repointed to a specific file: {imports}" + ) + + +def test_non_python_import_edge_is_not_repointed(tmp_path): + """#2072 review: the alias map is Python-only, but a non-Python import edge + whose dangling target coincides with a Python alias must NOT be repointed + onto a Python file (that would fabricate a cross-language import).""" + pkg = tmp_path / "src" / "pkg" + pkg.mkdir(parents=True) + (pkg / "__init__.py").write_text("") + (pkg / "mod.py").write_text("def f():\n return 1\n") + # Simulate a non-Python (C#) import edge whose target string collides with the + # Python alias `pkg_mod`, by hand-building the extraction the way extract emits. + result = extract([pkg / "__init__.py", pkg / "mod.py"], cache_root=tmp_path / "c", + root=tmp_path, parallel=False) + result["nodes"].append( + {"id": "app_cs", "label": "app.cs", "file_type": "code", "source_file": "app.cs"} + ) + result["edges"].append( + {"source": "app_cs", "target": "pkg_mod", "relation": "imports", + "confidence": "EXTRACTED", "source_file": "app.cs"} + ) + G = build_from_json(result, root=str(tmp_path)) + # The C# edge's target must remain the (dangling, dropped) `pkg_mod`, never + # repointed to the Python file node src_pkg_mod. + assert not any(v == "src_pkg_mod" and u == "app_cs" for _, u, v in _import_edges(G)), ( + "non-Python import edge was repointed onto a Python file (#2072 review)" + ) diff --git a/tests/test_stale_prune.py b/tests/test_stale_prune.py new file mode 100644 index 000000000..b4997b1ce --- /dev/null +++ b/tests/test_stale_prune.py @@ -0,0 +1,118 @@ +"""#2210: incremental extract's graph-layer prune must not evict ALIVE files. + +_stale_graph_sources compared stored graph source_file spellings against the +current scan with a raw string membership test: (a) no NFC/NFD normalization, +so a macOS NFD on-disk path never matched an NFC graph entry; (b) no liveness +check, so any membership miss was declared "deleted" and pruned even though +the file exists on disk and is in the scan. These tests exercise the NFC +normalization, the fail-closed liveness guard, and that genuinely-deleted +sources are still pruned. +""" +from __future__ import annotations +import json +import unicodedata + +from graphify.cli import _stale_graph_sources +from graphify.detect import detect + +NFC_NAME = unicodedata.normalize("NFC", "café.md") # café.md, composed +NFD_NAME = unicodedata.normalize("NFD", "café.md") # cafe + combining accent + + +def _write_graph(tmp_path, source_files: list[str]): + out = tmp_path / "graphify-out" + out.mkdir(exist_ok=True) + graph_path = out / "graph.json" + nodes = [ + {"id": f"n{i}", "label": f"node {i}", "source_file": sf} + for i, sf in enumerate(source_files) + ] + graph_path.write_text( + json.dumps({"nodes": nodes, "links": []}), encoding="utf-8" + ) + return graph_path + + +def _scan(tmp_path): + detection = detect(tmp_path) + seen = {f for flist in detection["files"].values() for f in flist} + seen.update(detection.get("unclassified", [])) + return detection, seen + + +def test_nfd_disk_nfc_graph_source_not_pruned(tmp_path): + """(a) NFD spelling on disk vs NFC spelling in the graph: NOT stale.""" + docs = tmp_path / "docs" + docs.mkdir() + (docs / NFD_NAME).write_text("# cafe notes\n\nhello\n", encoding="utf-8") + + graph_path = _write_graph(tmp_path, ["docs/" + NFC_NAME]) + detection, seen = _scan(tmp_path) + + stale = _stale_graph_sources(graph_path, tmp_path, seen, detection=detection) + assert stale == [] + + +def test_bare_basename_alive_elsewhere_not_pruned(tmp_path, capsys): + """(b) fail-closed: a legacy bare-basename source_file whose file is + alive at docs/café.md cannot be proven deleted — keep it.""" + docs = tmp_path / "docs" + docs.mkdir() + (docs / NFD_NAME).write_text("# cafe notes\n\nhello\n", encoding="utf-8") + + graph_path = _write_graph(tmp_path, [NFC_NAME]) # bare legacy spelling + detection, seen = _scan(tmp_path) + + stale = _stale_graph_sources(graph_path, tmp_path, seen, detection=detection) + assert stale == [] + + +def test_genuinely_deleted_source_still_pruned(tmp_path): + """(c) a source_file with no file on disk anywhere IS pruned.""" + docs = tmp_path / "docs" + docs.mkdir() + (docs / "keep.md").write_text("# keep\n\nstill here\n", encoding="utf-8") + + graph_path = _write_graph(tmp_path, ["docs/keep.md", "docs/gone.md"]) + detection, seen = _scan(tmp_path) + + stale = _stale_graph_sources(graph_path, tmp_path, seen, detection=detection) + assert stale == ["docs/gone.md"] + + +def test_alive_but_ignored_source_is_pruned(tmp_path): + """#1909 must keep working: an alive file excluded by ignore rules is + provably excluded, so its nodes ARE pruned.""" + docs = tmp_path / "docs" + docs.mkdir() + (docs / "keep.md").write_text("# keep\n\nstill here\n", encoding="utf-8") + (docs / "secret.md").write_text("# secret\n\nexcluded\n", encoding="utf-8") + (tmp_path / ".graphifyignore").write_text("docs/secret.md\n", encoding="utf-8") + + graph_path = _write_graph(tmp_path, ["docs/keep.md", "docs/secret.md"]) + detection, seen = _scan(tmp_path) + + stale = _stale_graph_sources(graph_path, tmp_path, seen, detection=detection) + assert stale == ["docs/secret.md"] + + +def test_alive_unproven_exclusion_kept_with_warning(tmp_path, capsys): + """Fail-closed: an alive in-root file missing from the corpus without + provable exclusion evidence is kept, and the keep is reported.""" + docs = tmp_path / "docs" + docs.mkdir() + (docs / "keep.md").write_text("# keep\n\nstill here\n", encoding="utf-8") + (docs / "other.md").write_text("# other\n\nalive\n", encoding="utf-8") + + graph_path = _write_graph(tmp_path, ["docs/keep.md", "docs/other.md"]) + detection, seen = _scan(tmp_path) + # Simulate a scan that lost docs/other.md for a reason that is NOT a + # provable exclusion (e.g. a walk error): drop it from seen and from + # the detection evidence. + seen = {s for s in seen if not s.endswith("other.md")} + detection = dict(detection, ignored=[], pruned_noise_dirs=[], skipped_sensitive=[]) + + stale = _stale_graph_sources(graph_path, tmp_path, seen, detection=detection) + assert stale == [] + err = capsys.readouterr().err + assert "fail-closed" in err diff --git a/tests/test_stat_index_portability.py b/tests/test_stat_index_portability.py new file mode 100644 index 000000000..51ae36a67 --- /dev/null +++ b/tests/test_stat_index_portability.py @@ -0,0 +1,243 @@ +"""#2199 — stat-index.json must be portable and self-pruning. + +The on-disk stat index used to key entries by resolved ABSOLUTE path, so a +corpus reached via a different absolute path (clone, move, second mount) got +0% cache hits (100% re-extraction), and entries for deleted files were never +pruned (unbounded growth). In-memory keys stay absolute; only the on-disk +form is relativized against the key anchor — mirroring the detect manifest's +_to_relative_for_storage/_to_absolute_from_storage round-trip. + +Also covers #2197 (cache.py portion): save_semantic_cache must normalize each +item's source_file (backslashes -> forward slashes, relativize when in-root) +before persisting, so a fragment carrying an absolute path (Windows detect() +output) cannot poison the cache. +""" +from __future__ import annotations + +import hashlib +import json +import os +import shutil +from pathlib import Path + +from graphify import cache + + +def _reset_stat_index(): + """The stat-index location/anchor are chosen once per process via module + globals (#1747/#2199). Reset them so each test sees a fresh-process + decision — same pattern as tests/test_extract_cache_location.py.""" + cache._stat_index_root = None + cache._stat_index_anchor = None + cache._stat_index = {} + cache._stat_index_dirty = False + + +def _stat_index_path(root: Path) -> Path: + return root / "graphify-out" / "cache" / "stat-index.json" + + +def _read_index(root: Path) -> dict: + return json.loads(_stat_index_path(root).read_text(encoding="utf-8")) + + +def _count_read_bytes(monkeypatch): + """Wrap Path.read_bytes with a call counter (file_hash's content read).""" + calls = {"n": 0} + orig = Path.read_bytes + + def counting(self): + calls["n"] += 1 + return orig(self) + + monkeypatch.setattr(Path, "read_bytes", counting) + return calls + + +def _fail_compute(p: Path) -> int: + raise AssertionError(f"word-count compute invoked for {p}; expected a warm stat hit") + + +def test_cache_hits_survive_corpus_move(tmp_path, monkeypatch): + """Run A under tmp/a, copy the corpus (with graphify-out/) to tmp/b: run B + must be 100% warm — zero content reads, zero word-count computes, digests + identical to run A.""" + _reset_stat_index() + a = tmp_path / "a" + a.mkdir() + (a / "f1.py").write_text("x = 1\n") + sub = a / "sub" + sub.mkdir() + (sub / "f2.md").write_text("hello world one two\n") + + digests_a = { + "f1.py": cache.file_hash(a / "f1.py", a), + "sub/f2.md": cache.file_hash(sub / "f2.md", a), + } + wc_a = cache.cached_word_count(a / "f1.py", a, lambda p: len(p.read_text().split())) + cache._flush_stat_index() + + on_disk = _read_index(a) + assert on_disk, "flush should have written entries" + for k in on_disk: + assert not os.path.isabs(k), f"absolute key leaked to disk: {k}" + assert "\\" not in k, f"non-portable separator in key: {k}" + assert set(on_disk) == {"f1.py", "sub/f2.md"} + + # Move the corpus (graphify-out/ rides along; copy2 preserves mtime_ns). + b = tmp_path / "b" + shutil.copytree(a, b, copy_function=shutil.copy2) + + _reset_stat_index() + reads = _count_read_bytes(monkeypatch) + assert cache.file_hash(b / "f1.py", b) == digests_a["f1.py"] + assert cache.file_hash(b / "sub" / "f2.md", b) == digests_a["sub/f2.md"] + assert cache.cached_word_count(b / "f1.py", b, _fail_compute) == wc_a + assert reads["n"] == 0, "moved corpus should be served entirely from the stat index" + + +def test_deleted_entries_are_pruned_on_flush(tmp_path): + _reset_stat_index() + a = tmp_path / "a" + a.mkdir() + f1 = a / "f1.py" + f1.write_text("x = 1\n") + f2 = a / "f2.py" + f2.write_text("y = 2\n") + cache.file_hash(f1, a) + cache.file_hash(f2, a) + cache._flush_stat_index() + assert set(_read_index(a)) == {"f1.py", "f2.py"} + + f2.unlink() + _reset_stat_index() + # Bump f1's mtime so the re-hash dirties the index and a flush is written. + os.utime(f1, ns=(f1.stat().st_atime_ns, f1.stat().st_mtime_ns + 1_000_000)) + cache.file_hash(f1, a) + cache._flush_stat_index() + + on_disk = _read_index(a) + assert set(on_disk) == {"f1.py"}, "deleted f2.py should have been pruned" + assert not os.path.isabs(next(iter(on_disk))) + + +def test_legacy_absolute_index_migrates_gracefully(tmp_path, monkeypatch): + """A pre-#2199 index keyed by absolute paths still HITS on the unmoved + root, and the first flush prunes dead entries and rewrites live keys + relative (self-heals).""" + _reset_stat_index() + a = tmp_path / "a" + a.mkdir() + f1 = a / "f1.py" + f1.write_text("x = 1\n") + st = f1.stat() + salt = "f1.py" + digest = hashlib.sha256(f1.read_bytes() + b"\x00" + salt.encode()).hexdigest() + + dead = tmp_path / "dead" # never created + legacy = { + str(f1.resolve()): {"size": st.st_size, "mtime_ns": st.st_mtime_ns, + "hashes": {salt: digest}}, + str(dead / "x.py"): {"size": 1, "mtime_ns": 1, "hashes": {"x.py": "aa"}}, + str(dead / "y.py"): {"size": 2, "mtime_ns": 2, "hashes": {"y.py": "bb"}}, + } + p = _stat_index_path(a) + p.parent.mkdir(parents=True) + p.write_text(json.dumps(legacy), encoding="utf-8") + + reads = _count_read_bytes(monkeypatch) + assert cache.file_hash(f1, a) == digest + assert reads["n"] == 0, "legacy absolute key should still serve a warm hit" + + # Force a write so the self-heal is observable (a pure warm run leaves the + # index clean and flush is a no-op by design). + cache._stat_index_dirty = True + cache._flush_stat_index() + + on_disk = _read_index(a) + assert set(on_disk) == {"f1.py"}, "dead absolute keys should be pruned" + assert on_disk["f1.py"]["hashes"][salt] == digest + + +def test_out_of_root_key_round_trips_absolute(tmp_path, monkeypatch): + _reset_stat_index() + a = tmp_path / "a" + a.mkdir() + outside = tmp_path / "outside.txt" + outside.write_text("out of root\n") + + d1 = cache.file_hash(outside, a) + cache._flush_stat_index() + + on_disk = _read_index(a) + assert set(on_disk) == {str(outside.resolve())}, "out-of-root key must stay absolute" + + _reset_stat_index() + reads = _count_read_bytes(monkeypatch) + assert cache.file_hash(outside, a) == d1 + assert reads["n"] == 0, "second call should be a stat hit" + + +def test_relative_key_wins_over_colliding_legacy_absolute(tmp_path): + """When an old absolute key and a new relative key resolve to the same + file, the relative (new-format) entry wins on load.""" + _reset_stat_index() + a = tmp_path / "a" + a.mkdir() + f1 = a / "f1.py" + f1.write_text("x = 1\n") + p = _stat_index_path(a) + p.parent.mkdir(parents=True) + p.write_text(json.dumps({ + str(f1.resolve()): {"size": 1, "mtime_ns": 1, "hashes": {"f1.py": "legacy"}}, + "f1.py": {"size": 2, "mtime_ns": 2, "hashes": {"f1.py": "fresh"}}, + }), encoding="utf-8") + + cache._ensure_stat_index(a) + assert cache._stat_index[str(f1.resolve())]["hashes"]["f1.py"] == "fresh" + + +def test_semantic_cache_normalizes_absolute_source_file(tmp_path): + """#2197: an item whose source_file is absolute is persisted root-relative + posix, and the caller's dict is not mutated.""" + _reset_stat_index() + root = tmp_path / "corpus" + root.mkdir() + f = root / "m.py" + f.write_text("x = 1\n") + + node = {"id": "m.x", "type": "variable", "source_file": str(f.resolve())} + saved = cache.save_semantic_cache([node], [], root=root) + assert saved == 1 + assert node["source_file"] == str(f.resolve()), "caller's dict must not be mutated" + + entries = list((root / "graphify-out" / "cache" / "semantic").glob("*.json")) + assert len(entries) == 1 + persisted = json.loads(entries[0].read_text(encoding="utf-8")) + assert persisted["nodes"][0]["source_file"] == "m.py" + + # Replay resolves back to the same absolute shape a fresh extraction has. + _, _, _, uncached = cache.check_semantic_cache([str(f)], root=root) + assert uncached == [] + + +def test_semantic_cache_normalizes_backslash_poisoned_source_file(tmp_path): + """A Windows-shaped absolute source_file (backslash separators) must be + slash-normalized and relativized instead of being skipped/persisted raw.""" + _reset_stat_index() + root = tmp_path / "corpus" + root.mkdir() + sub = root / "sub" + sub.mkdir() + f = sub / "n.py" + f.write_text("y = 2\n") + + poisoned = str(root.resolve()) + "\\sub\\n.py" + node = {"id": "n.y", "type": "variable", "source_file": poisoned} + saved = cache.save_semantic_cache([node], [], root=root) + assert saved == 1 + + entries = list((root / "graphify-out" / "cache" / "semantic").glob("*.json")) + assert len(entries) == 1 + persisted = json.loads(entries[0].read_text(encoding="utf-8")) + assert persisted["nodes"][0]["source_file"] == "sub/n.py" diff --git a/tests/test_swift_builtin_noise.py b/tests/test_swift_builtin_noise.py new file mode 100644 index 000000000..bea4ae000 --- /dev/null +++ b/tests/test_swift_builtin_noise.py @@ -0,0 +1,143 @@ +"""Swift/Foundation/SwiftUI builtins must not become god nodes or bind to user symbols. + +#2147: `_LANGUAGE_BUILTIN_GLOBALS` and `_BUILTIN_NOISE_LABELS` covered only +JS/TS and Python, so on Swift codebases framework types (`Foundation`, +`NSLock`, `View`, `Data`, `Sendable`, ...) ranked as god nodes and the Swift +member-call resolver could bind a builtin-typed receiver (`let d: Data`) to a +same-named user symbol in another file — the same phantom-edge shape #1726 +fixed for TypeScript. +""" +import networkx as nx +import pytest + +from graphify.analyze import god_nodes +from graphify.extract import extract + + +def _labels_by_id(r): + return {n["id"]: n.get("label") for n in r["nodes"]} + + +@pytest.mark.parametrize("builtin_label", [ + "Foundation", "SwiftUI", "NSLock", "Data", "View", "Sendable", "Codable", + "DispatchQueue", "Color", +]) +def test_god_nodes_excludes_swift_builtin_labels(builtin_label: str) -> None: + """Swift framework symbols must be filtered from god_nodes output. + + Constructs a graph where the builtin-labelled node has the highest degree + (the #2147 report shape: `Foundation` at 105 edges, `NSLock` at 102) and a + real project abstraction has lower degree. The builtin must be excluded and + the project symbol kept. + + Args: + builtin_label: The Swift/Foundation/SwiftUI label to test (parametrized). + """ + G = nx.Graph() + G.add_node( + "real_node", + label="AudioStreamer", + source_file="Sources/AudioStreamer.swift", + file_type="code", + source_location="L1", + ) + G.add_node( + "builtin_node", + label=builtin_label, + source_file="", + file_type="code", + source_location="", + ) + for i in range(20): + peer = f"user_type_{i}" + G.add_node( + peer, + label=f"Feature{i}", + source_file=f"Sources/Feature{i}.swift", + file_type="code", + source_location="L1", + ) + G.add_edge( + peer, + "builtin_node", + relation="references", + confidence="EXTRACTED", + source_file=f"Sources/Feature{i}.swift", + weight=1.0, + ) + G.add_edge( + "real_node", + "user_type_0", + relation="calls", + confidence="EXTRACTED", + source_file="Sources/AudioStreamer.swift", + weight=1.0, + ) + + result = god_nodes(G, top_n=10) + result_ids = [r["id"] for r in result] + + assert "builtin_node" not in result_ids, ( + f"god_nodes() should filter Swift builtin '{builtin_label}' " + f"but it appeared in the result: {result}" + ) + assert "real_node" in result_ids, ( + f"god_nodes() should include project symbol 'AudioStreamer' " + f"but it was absent: {result}" + ) + + +def test_swift_builtin_receiver_does_not_bind_to_user_symbol(tmp_path): + # #2147 (same shape as #1726 for TS): a receiver typed with a builtin + # (`let payload: Data`) must not have its member calls bound to a user type + # that happens to be named `Data` in another file. + (tmp_path / "Model.swift").write_text( + "class Data {\n" + " func append(_ s: String) {}\n" + "}\n") + (tmp_path / "Uploader.swift").write_text( + "class Uploader {\n" + " let payload: Data = Data()\n" + " func send() {\n" + " payload.append(\"x\")\n" + " }\n" + "}\n") + r = extract(sorted(tmp_path.glob("*.swift")), cache_root=tmp_path, parallel=False) + lbl = _labels_by_id(r) + by_id = {n["id"]: n for n in r["nodes"]} + data_ids = [n["id"] for n in r["nodes"] + if n.get("label") == "Data" + and str(n.get("source_file", "")).endswith("Model.swift")] + assert data_ids, "the user class Data must still exist as a node" + for e in r["edges"]: + if e.get("target") in data_ids and e.get("relation") in ("calls", "references") \ + and e.get("context") == "call": + src_sf = str(by_id.get(e["source"], {}).get("source_file", "")) + assert not src_sf.endswith("Uploader.swift"), ( + f"builtin-typed receiver bound to user Data: " + f"{lbl.get(e['source'])!r} -> Data ({e})" + ) + + +def test_swift_user_receiver_type_still_resolves(tmp_path): + # Guard must be a no-op for genuine user types: a member call on a + # user-typed property still resolves cross-file (#1356 inference table). + (tmp_path / "Engine.swift").write_text( + "class AudioEngine {\n" + " func play() {}\n" + "}\n") + (tmp_path / "Player.swift").write_text( + "class Player {\n" + " let engine: AudioEngine = AudioEngine()\n" + " func start() {\n" + " engine.play()\n" + " }\n" + "}\n") + r = extract(sorted(tmp_path.glob("*.swift")), cache_root=tmp_path, parallel=False) + lbl = _labels_by_id(r) + resolved = { + (lbl.get(e["source"]), lbl.get(e["target"])) + for e in r["edges"] + if e.get("context") == "call" and "play" in str(lbl.get(e.get("target"), "")).lower() + } + assert resolved, "user-typed member call must still resolve cross-file" diff --git a/tests/test_swift_computed_properties.py b/tests/test_swift_computed_properties.py new file mode 100644 index 000000000..dd78d7482 --- /dev/null +++ b/tests/test_swift_computed_properties.py @@ -0,0 +1,86 @@ +"""Regression tests for #2181. + +Swift computed properties (`var body: some View { … }`) and observed +properties (`willSet` / `didSet`) carry a body. Before the fix the Swift +extractor only recognised function/init/deinit/subscript as callables, so +those properties produced no node and their bodies were never walked -- which +for SwiftUI erased the entire view layer (`body` is a computed property). +""" +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path + +from graphify.extract import extract_swift + + +def _labels(result): + return [n["label"] for n in result["nodes"]] + + +def _rel(result, relation): + return [e for e in result["edges"] if e["relation"] == relation] + + +class TestSwiftComputedProperties(unittest.TestCase): + def _extract(self, src: str) -> dict: + with tempfile.TemporaryDirectory() as d: + p = Path(d) / "View.swift" + p.write_text(src, encoding="utf-8") + return extract_swift(p) + + def test_computed_property_emits_node_and_walks_body(self): + r = self._extract( + "struct PlayerScrubber: View {\n" + " var body: some View {\n" + " VStack { doTap() }\n" + " }\n" + " var toggled: Int { 1 }\n" + " func doTap() {}\n" + "}\n" + ) + labels = _labels(r) + # Both computed properties become nodes... + self.assertIn(".body", labels) + self.assertIn(".toggled", labels) + # ...and the call inside `body` is attributed to the body node, not lost. + call_pairs = {(e["source"], e["target"]) for e in _rel(r, "calls")} + body_nid = next(n["id"] for n in r["nodes"] if n["label"] == ".body") + dotap_nid = next(n["id"] for n in r["nodes"] if n["label"] == ".doTap()") + self.assertIn((body_nid, dotap_nid), call_pairs, + "call inside computed `body` was not captured") + + def test_stored_property_not_emitted_as_member_but_keeps_type_ref(self): + # A plain stored property has no body block: it must NOT become a + # function-like node, but its type must still produce a references edge. + r = self._extract( + "struct S {\n" + " var vm: ViewModel\n" + "}\n" + ) + self.assertNotIn(".vm", _labels(r)) + ref_targets = {n["label"] + for e in _rel(r, "references") + for n in r["nodes"] if n["id"] == e["target"]} + self.assertIn("ViewModel", ref_targets) + + def test_observed_property_body_is_walked(self): + # willSet/didSet observers also carry a body whose calls used to vanish. + r = self._extract( + "class M {\n" + " var score: Int = 0 {\n" + " didSet { react() }\n" + " }\n" + " func react() {}\n" + "}\n" + ) + self.assertIn(".score", _labels(r)) + call_pairs = {(e["source"], e["target"]) for e in _rel(r, "calls")} + score_nid = next(n["id"] for n in r["nodes"] if n["label"] == ".score") + react_nid = next(n["id"] for n in r["nodes"] if n["label"] == ".react()") + self.assertIn((score_nid, react_nid), call_pairs) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_typescript_module_extensions.py b/tests/test_typescript_module_extensions.py index 41091a552..33c8493d2 100644 --- a/tests/test_typescript_module_extensions.py +++ b/tests/test_typescript_module_extensions.py @@ -78,6 +78,13 @@ def test_cts_uses_the_typescript_grammar(tmp_path): } +def test_uppercase_typescript_extensions_use_typescript_grammar(tmp_path): + for ext in (".TS", ".TSX", ".MTS", ".CTS"): + labels = _labels(_extract(tmp_path, ext)) + assert any("Mode" in label for label in labels), f"TS `type` alias missing for {ext}" + assert any("Options" in label for label in labels), f"TS `interface` missing for {ext}" + + def test_mts_cts_route_to_extract_js(): from graphify.extract import _DISPATCH, extract_js assert _DISPATCH.get(".mts") is extract_js diff --git a/tests/test_uninstall_scope.py b/tests/test_uninstall_scope.py new file mode 100644 index 000000000..9102cdfab --- /dev/null +++ b/tests/test_uninstall_scope.py @@ -0,0 +1,113 @@ +"""Scope regression tests for the uninstall API trap (issue #2215). + +`X_uninstall(project_dir)` used to delete the USER-GLOBAL skill tree because +`_platform_skill_destination` honors ``project_dir`` only when ``project=True``. +These tests pin the fixed contract: + +- bare call -> global skill removed (CLI behavior unchanged) +- fn(pd) -> project-scoped, global untouched (trap closed) +- fn(pd, project=True) -> project only +- fn(pd, remove_user_skill=True) -> global removed, project tree untouched +- `graphify uninstall --project` for codebuddy no longer nukes the global skill +""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from graphify.install import ( + _project_uninstall, + claude_uninstall, + codebuddy_uninstall, + gemini_uninstall, +) + +PLATFORMS = [ + pytest.param(claude_uninstall, "claude", ".claude", id="claude"), + pytest.param(gemini_uninstall, "gemini", ".gemini", id="gemini"), + pytest.param(codebuddy_uninstall, "codebuddy", ".codebuddy", id="codebuddy"), +] + + +def _plant_skill_tree(root: Path, dot_dir: str) -> Path: + """Create //skills/graphify/{SKILL.md, references/x.md, .graphify_version}.""" + skill_dir = root / dot_dir / "skills" / "graphify" + (skill_dir / "references").mkdir(parents=True) + (skill_dir / "SKILL.md").write_text("# graphify skill\n", encoding="utf-8") + (skill_dir / "references" / "x.md").write_text("ref\n", encoding="utf-8") + (skill_dir / ".graphify_version").write_text("0.0.0-test", encoding="utf-8") + return skill_dir + + +@pytest.mark.parametrize("uninstall_fn,platform,dot_dir", PLATFORMS) +def test_project_dir_call_never_touches_global(uninstall_fn, platform, dot_dir, tmp_path): + """fn(project_dir) removes only the project skill tree (#2215 trap closed).""" + global_tree = _plant_skill_tree(Path.home(), dot_dir) + proj_dir = tmp_path / "proj" + project_tree = _plant_skill_tree(proj_dir, dot_dir) + + uninstall_fn(proj_dir) + + assert (global_tree / "SKILL.md").exists(), "global skill deleted by project-scoped uninstall" + assert (global_tree / "references" / "x.md").exists() + assert (global_tree / ".graphify_version").exists() + assert not (project_tree / "SKILL.md").exists() + assert not project_tree.exists() + + +@pytest.mark.parametrize("uninstall_fn,platform,dot_dir", PLATFORMS) +def test_bare_call_still_removes_global(uninstall_fn, platform, dot_dir, tmp_path, monkeypatch): + """fn() with no args keeps the historical CLI behavior: global skill removed.""" + global_tree = _plant_skill_tree(Path.home(), dot_dir) + cwd = tmp_path / "empty-cwd" + cwd.mkdir() + monkeypatch.chdir(cwd) + + uninstall_fn() + + assert not (global_tree / "SKILL.md").exists() + assert not global_tree.exists() + + +@pytest.mark.parametrize("uninstall_fn,platform,dot_dir", PLATFORMS) +def test_remove_user_skill_opt_in_with_project_dir(uninstall_fn, platform, dot_dir, tmp_path): + """fn(pd, remove_user_skill=True) removes the global skill, leaves the project tree.""" + global_tree = _plant_skill_tree(Path.home(), dot_dir) + proj_dir = tmp_path / "proj" + project_tree = _plant_skill_tree(proj_dir, dot_dir) + + uninstall_fn(proj_dir, remove_user_skill=True) + + assert not (global_tree / "SKILL.md").exists() + assert not global_tree.exists() + assert (project_tree / "SKILL.md").exists() + assert (project_tree / "references" / "x.md").exists() + + +@pytest.mark.parametrize("uninstall_fn,platform,dot_dir", PLATFORMS) +def test_project_true_removes_only_project_tree(uninstall_fn, platform, dot_dir, tmp_path): + """fn(pd, project=True) removes only the project skill tree.""" + global_tree = _plant_skill_tree(Path.home(), dot_dir) + proj_dir = tmp_path / "proj" + project_tree = _plant_skill_tree(proj_dir, dot_dir) + + uninstall_fn(proj_dir, project=True) + + assert (global_tree / "SKILL.md").exists() + assert not (project_tree / "SKILL.md").exists() + assert not project_tree.exists() + + +def test_project_uninstall_codebuddy_spares_global(tmp_path): + """`graphify uninstall --project` (codebuddy branch) must not delete ~/.codebuddy (#2215).""" + global_tree = _plant_skill_tree(Path.home(), ".codebuddy") + proj_dir = tmp_path / "proj" + project_tree = _plant_skill_tree(proj_dir, ".codebuddy") + + _project_uninstall("codebuddy", proj_dir) + + assert (global_tree / "SKILL.md").exists(), "CLI --project uninstall deleted the global codebuddy skill" + assert (global_tree / ".graphify_version").exists() + assert not (project_tree / "SKILL.md").exists() + assert not project_tree.exists() diff --git a/tests/test_validate.py b/tests/test_validate.py index ea865faeb..013fa8721 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -87,6 +87,27 @@ def test_assert_valid_passes_silently(): assert_valid(VALID) # should not raise +def test_legacy_aliases_valid_after_build_canonicalization(): + # #2194: build_from_json folds legacy aliases (name->label, + # path->source_file, type->relation, confidence_score->confidence) in + # place BEFORE validation, so an alias-only extraction that fails + # validation raw is fully schema-valid after canonicalization. + from graphify.build import build_from_json + data = { + "nodes": [ + {"id": "n1", "name": "Foo", "path": "a/b.md", "file_type": "concept"}, + {"id": "n2", "label": "Bar", "file_type": "code", "source_file": "bar.py"}, + ], + "edges": [ + {"source": "n1", "target": "n2", "type": "references", + "confidence_score": 0.9, "source_file": "a/b.md"}, + ], + } + assert any("missing required field" in e for e in validate_extraction(data)) + build_from_json(data) + assert validate_extraction(data) == [] + + def test_non_hashable_node_id_reported_not_raised(): # A malformed LLM extraction can emit a list-valued id. The validator must # report it as an error string (its documented contract) rather than crash diff --git a/tests/test_watch.py b/tests/test_watch.py index fb9868677..f2bec415f 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -160,6 +160,275 @@ def test_graphify_root_preserves_relative_when_invoked_with_relative_path(tmp_pa ) +def test_rebuild_code_writes_community_name(tmp_path): + """#1808: `graphify update` / _rebuild_code must forward community_labels to + to_json, so graph.json nodes carry a human-readable community_name (hub-derived + for a code-only rebuild) — not just a numeric community id. Before the fix, + _rebuild_code called to_json without community_labels, so the labels a + cluster-only pass writes were stripped again on every incremental rebuild.""" + import json + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "a.py").write_text( + "def alpha():\n return beta()\n\ndef beta():\n return 1\n", encoding="utf-8" + ) + (corpus / "b.py").write_text( + "import a\n\ndef gamma():\n return a.alpha()\n", encoding="utf-8" + ) + assert _rebuild_code(corpus, acquire_lock=False) is True + + graph = json.loads((corpus / "graphify-out" / "graph.json").read_text(encoding="utf-8")) + clustered = [n for n in graph["nodes"] if n.get("community") is not None] + assert clustered, "expected clustered nodes in the rebuilt graph" + assert all(n.get("community_name") for n in clustered), ( + "clustered nodes missing community_name — the update rebuild stripped the " + "labels that cluster-only writes (#1808)" + ) + + +def test_rebuild_code_drops_labels_whose_community_changed(tmp_path): + """An incremental rebuild must not reuse a saved label for a community whose + membership changed. Labels are keyed by cid, but re-clustering reassigns cids, + so after new files land cid N can cover a different community and its old name + is then simply wrong. cluster-only guards this with the `.sig` membership + fingerprints; _rebuild_code ignored them and hub-filled only *missing* labels, + so stale names survived and were written back to labels.json as if current.""" + import json + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "a.py").write_text( + "def alpha():\n return beta()\n\ndef beta():\n return 1\n", encoding="utf-8" + ) + assert _rebuild_code(corpus, acquire_lock=False) is True + + out = corpus / "graphify-out" + labels_file = out / ".graphify_labels.json" + sig_file = out / ".graphify_labels.json.sig" + assert sig_file.exists(), "rebuild must persist membership signatures beside labels" + + # Stand in for an LLM naming pass: give every community a distinctive name, + # leaving the signatures untouched so they still describe THIS clustering. + labels = json.loads(labels_file.read_text(encoding="utf-8")) + assert labels, "expected the first rebuild to write community labels" + labels_file.write_text( + json.dumps({cid: f"Named-{cid}" for cid in labels}), encoding="utf-8" + ) + + # Grow the corpus so clustering changes, then rebuild incrementally. + for name in ("b.py", "c.py", "d.py"): + (corpus / name).write_text( + f"def {name[0]}_one():\n return {name[0]}_two()\n\n" + f"def {name[0]}_two():\n return 2\n", + encoding="utf-8", + ) + assert _rebuild_code(corpus, acquire_lock=False) is True + + graph = json.loads((out / "graph.json").read_text(encoding="utf-8")) + after = json.loads(labels_file.read_text(encoding="utf-8")) + sigs = json.loads(sig_file.read_text(encoding="utf-8")) + + communities = {} + for node in graph["nodes"]: + cid = node.get("community") + if cid is not None: + communities.setdefault(str(cid), []).append(node["id"]) + + from graphify.cluster import community_member_sigs + expected = { + str(cid): sig + for cid, sig in community_member_sigs( + {int(c): m for c, m in communities.items()} + ).items() + } + assert sigs == expected, ( + "signatures must be rewritten in step with the labels; a drifting sidecar " + "leaves the staleness guard nothing accurate to check against" + ) + + # Any surviving "Named-N" must sit on a community that genuinely did not change. + for cid, name in after.items(): + if name.startswith("Named-"): + assert cid in communities, f"label kept for vanished community {cid}" + assert sigs[cid] == expected[cid], ( + f"community {cid} kept the stale label {name!r} after its " + f"membership changed" + ) + + for node in graph["nodes"]: + cid = node.get("community") + if cid is not None and node.get("community_name", "").startswith("Named-"): + assert sigs[str(cid)] == expected[str(cid)], ( + f"node {node['id']} carries stale community_name " + f"{node['community_name']!r}" + ) + + +def test_rebuild_code_keeps_a_visualization_when_over_the_viz_cap(tmp_path, monkeypatch): + """Crossing the viz node limit must not leave the project with no graph.html. + _rebuild_code used to unlink the existing file and write nothing, so a repo + that grew past the cap silently lost its visualization — and the file was + already gone by the time the user read the message. The export path falls + back to the community-aggregation view in exactly this case; the incremental + path should too, so the artifact stays both current and present.""" + import json + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + # Several *disconnected* clusters: the aggregator declines to render a + # single-community meta-graph, so a ring of mutually-importing modules + # would collapse to one community and exercise the wrong path. + for g in range(4): + for i in range(3): + other = (i + 1) % 3 + (corpus / f"g{g}_m{i}.py").write_text( + f"import g{g}_m{other}\n\n" + + "".join(f"def g{g}_f{i}_{j}():\n return {j}\n\n" for j in range(4)), + encoding="utf-8", + ) + assert _rebuild_code(corpus, acquire_lock=False) is True + html = corpus / "graphify-out" / "graph.html" + assert html.exists(), "expected a normal (under-cap) rebuild to write graph.html" + before = html.read_text(encoding="utf-8") + + # Drop the cap below the graph's size but above its community count — the + # real shape of this bug. (A cap under the community count would make the + # aggregated meta-graph breach it too, which is a different situation.) + graph = json.loads((corpus / "graphify-out" / "graph.json").read_text(encoding="utf-8")) + communities = {n.get("community") for n in graph["nodes"] if n.get("community") is not None} + cap = (len(communities) + len(graph["nodes"])) // 2 + assert len(communities) < cap < len(graph["nodes"]), "test corpus cannot exercise the cap" + monkeypatch.setenv("GRAPHIFY_VIZ_NODE_LIMIT", str(cap)) + (corpus / "g9_extra.py").write_text("def extra():\n return 1\n", encoding="utf-8") + assert _rebuild_code(corpus, acquire_lock=False) is True + + assert html.exists(), ( + "graph.html was deleted when the graph exceeded the viz cap — the " + "incremental rebuild must fall back to the aggregated view like export does" + ) + after = html.read_text(encoding="utf-8") + assert after != before, "graph.html must be re-rendered, not left stale" + + # And the documented kill switch still means "no viz", not "aggregate". + monkeypatch.setenv("GRAPHIFY_VIZ_NODE_LIMIT", "0") + (corpus / "g9_extra2.py").write_text("def extra2():\n return 2\n", encoding="utf-8") + assert _rebuild_code(corpus, acquire_lock=False) is True + assert not html.exists(), "GRAPHIFY_VIZ_NODE_LIMIT=0 must disable the HTML viz outright" + + +def test_update_rebuilds_with_nested_star_gitignore(tmp_path): + """#1880: `graphify update` must not emit 0 nodes (and then refuse to + overwrite) just because the source tree has a nested `.gitignore` with a + broad pattern. This was the 0.9.15 symptom of the #1847/#1873 subtree-scoping + bug: a nested bare `*` zeroed the re-scan, update built 0 nodes, and the + shrink-guard refused. With scoping fixed the rebuild sees the real files.""" + import json + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + (corpus / "src").mkdir(parents=True) + (corpus / "src" / "a.py").write_text( + "from src.b import Base\nclass App(Base):\n def run(self): return 1\n", encoding="utf-8" + ) + (corpus / "src" / "b.py").write_text("class Base: pass\n", encoding="utf-8") + (corpus / "main.py").write_text("def top(): return 2\n", encoding="utf-8") + # a common scratch-dir idiom deeper in the tree: ignore everything HERE only + (corpus / "scratch").mkdir() + (corpus / "scratch" / ".gitignore").write_text("*\n", encoding="utf-8") + (corpus / "scratch" / "junk.py").write_text("x = 1\n", encoding="utf-8") + + assert _rebuild_code(corpus, acquire_lock=False) is True + + graph = json.loads((corpus / "graphify-out" / "graph.json").read_text(encoding="utf-8")) + sources = {n.get("source_file", "") for n in graph["nodes"]} + assert graph["nodes"], "update produced 0 nodes on a tree with a nested '*' gitignore (#1880)" + assert any("src/a.py" in s for s in sources) and any("main.py" in s for s in sources) + # the nested-ignored scratch file stays out (scoped correctly, not tree-wide) + assert not any("scratch/junk.py" in s for s in sources) + + +def test_update_discovers_newly_added_files_and_dirs(tmp_path): + """#1837: after an initial build, a plain `graphify update` (full re-scan, no + change-list) must discover brand-new files AND new directories. The reported + silent no-op was the #1873 nested-gitignore scoping bug zeroing the re-scan; + this pins the build -> add -> update -> discovered sequence the earlier test + (single build) did not cover, with a nested `*` scratch dir as a guard.""" + import json + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + (corpus / "src").mkdir(parents=True) + (corpus / "src" / "a.py").write_text("def alpha(): return 1\n", encoding="utf-8") + assert _rebuild_code(corpus, acquire_lock=False) is True + + # Add a brand-new file and a brand-new nested directory after the first build, + # plus a scratch dir that ignores only itself. + (corpus / "src" / "new.py").write_text("def added(): return 2\n", encoding="utf-8") + (corpus / "monitor").mkdir() + (corpus / "monitor" / "dash.py").write_text("def board(): return 3\n", encoding="utf-8") + (corpus / "scratch").mkdir() + (corpus / "scratch" / ".gitignore").write_text("*\n", encoding="utf-8") + (corpus / "scratch" / "junk.py").write_text("x = 1\n", encoding="utf-8") + + assert _rebuild_code(corpus, acquire_lock=False) is True + + sources = {n.get("source_file", "") for n in + json.loads((corpus / "graphify-out" / "graph.json").read_text())["nodes"]} + assert any("src/new.py" in s for s in sources), "new file not discovered by update (#1837)" + assert any("monitor/dash.py" in s for s in sources), "new directory not discovered (#1837)" + assert not any("scratch/junk.py" in s for s in sources) + + +def test_rebuild_honors_persisted_excludes(tmp_path): + """#1886: `--exclude` recorded at extract time must survive into update/watch/ + hook rebuilds. Before the fix only the initial scan applied the excludes, so + the first rebuild silently re-indexed the excluded paths. _rebuild_code now + reads the persisted build config and re-applies them.""" + import json + from graphify.watch import _rebuild_code, _write_build_config + + corpus = tmp_path / "corpus" + (corpus / "src").mkdir(parents=True) + (corpus / "vendor").mkdir() + (corpus / "src" / "app.py").write_text("def keep(): return 1\n", encoding="utf-8") + (corpus / "main.py").write_text("def top(): return 2\n", encoding="utf-8") + (corpus / "vendor" / "lib.py").write_text("def vendored(): pass\n", encoding="utf-8") + _write_build_config(corpus / "graphify-out", excludes=["vendor"]) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + graph = json.loads((corpus / "graphify-out" / "graph.json").read_text(encoding="utf-8")) + sources = {n.get("source_file", "") for n in graph["nodes"]} + assert any("src/app.py" in s for s in sources) and any("main.py" in s for s in sources) + assert not any("vendor/lib.py" in s for s in sources), ( + "rebuild silently re-included an excluded path (#1886)" + ) + + +def test_rebuild_honors_persisted_no_gitignore(tmp_path): + import json + from graphify.watch import _rebuild_code, _write_build_config + + corpus = tmp_path / "corpus" + generated = corpus / "generated" + generated.mkdir(parents=True) + (corpus / ".gitignore").write_text("generated/\n") + (generated / "gen.py").write_text("def generated(): return 1\n") + _write_build_config( + corpus / "graphify-out", excludes=None, gitignore=False + ) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + graph = json.loads((corpus / "graphify-out" / "graph.json").read_text()) + sources = {Path(str(node.get("source_file", ""))).as_posix() for node in graph["nodes"]} + assert any(source.endswith("generated/gen.py") for source in sources) + + def test_graphify_root_preserves_absolute_when_user_supplied(tmp_path): """When the caller supplies an absolute path, ``.graphify_root`` stores that absolute form verbatim — preserving explicit-absolute intent.""" @@ -176,6 +445,56 @@ def test_graphify_root_preserves_absolute_when_user_supplied(tmp_path): ) +def test_rebuild_code_deleted_cwd_without_repo_root_returns_false(tmp_path, monkeypatch, capsys): + """Detached hooks can inherit a CWD that no longer exists. + + Without GRAPHIFY_REPO_ROOT, the rebuild should fail cleanly before creating + relative graphify-out queue/lock files. + """ + from graphify.watch import _rebuild_code + + old_cwd = Path.cwd() + gone = tmp_path / "gone" + gone.mkdir() + monkeypatch.delenv("GRAPHIFY_REPO_ROOT", raising=False) + + os.chdir(gone) + gone.rmdir() + try: + assert _rebuild_code(Path("."), changed_paths=[Path("lib.py")]) is False + finally: + os.chdir(old_cwd) + + out = capsys.readouterr().out + assert "current working directory no longer exists" in out + + +def test_rebuild_code_deleted_cwd_uses_graphify_repo_root(tmp_path, monkeypatch): + """GRAPHIFY_REPO_ROOT lets detached hook rebuilds recover from a deleted CWD.""" + from graphify.watch import _rebuild_code + + old_cwd = Path.cwd() + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "lib.py").write_text("def f(): pass\n", encoding="utf-8") + gone = tmp_path / "gone" + gone.mkdir() + monkeypatch.setenv("GRAPHIFY_REPO_ROOT", str(corpus)) + + os.chdir(gone) + gone.rmdir() + try: + assert _rebuild_code( + Path("."), + changed_paths=[Path("lib.py")], + no_cluster=True, + ) is True + assert Path.cwd().resolve() == corpus.resolve() + assert (corpus / "graphify-out" / "graph.json").exists() + finally: + os.chdir(old_cwd) + + def test_rebuild_code_evicts_nodes_from_deleted_files(tmp_path): """#1007: graphify update (_rebuild_code with no changed_paths) must remove nodes and edges from files deleted since the last run.""" @@ -226,6 +545,123 @@ def _add_unrelated_semantic_pair(graph_path): graph_path.write_text(json.dumps(data), encoding="utf-8") +@pytest.mark.parametrize( + "changed_paths", + [None, [Path("doc.md")]], + ids=["full-update", "incremental-doc-update"], +) +def test_rebuild_code_preserves_hyperedges_for_rebuilt_surviving_source( + tmp_path, changed_paths +): + """#1755: AST-only updates must not drop semantic hyperedges whose members survive.""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "doc.md").write_text( + "# Design\n\n## Flow\n\nDetails.\n", encoding="utf-8" + ) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + assert {"doc", "doc_design"} <= {node["id"] for node in data["nodes"]} + data["hyperedges"] = [{ + "id": "doc_flow_group", + "label": "Doc flow group", + "nodes": ["doc", "doc_design"], + "relation": "implements", + "confidence": "EXTRACTED", + "confidence_score": 1.0, + "source_file": "doc.md", + }] + graph_path.write_text(json.dumps(data), encoding="utf-8") + + assert _rebuild_code( + corpus, + changed_paths=changed_paths, + no_cluster=True, + acquire_lock=False, + ) is True + + after = json.loads(graph_path.read_text(encoding="utf-8")) + assert after["hyperedges"] == [{ + "id": "doc_flow_group", + "label": "Doc flow group", + "nodes": ["doc", "doc_design"], + "relation": "implements", + "confidence": "EXTRACTED", + "confidence_score": 1.0, + "source_file": "doc.md", + }] + + +@pytest.mark.parametrize( + "changed_paths", + [None, [Path("auth.md")]], + ids=["full-update", "incremental-doc-update"], +) +def test_rebuild_code_preserves_semantic_edges_from_reextracted_doc( + tmp_path, changed_paths +): + """#1865: AST-only updates must not evict semantic edges whose source_file + is a re-extracted document; only that source's AST-tier edges are replaced.""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "auth.md").write_text( + "# Token Validation\n\nVerifies bearer tokens.\n", encoding="utf-8" + ) + (corpus / "login.md").write_text( + "# Session Verification\n\nVerifies login sessions.\n", encoding="utf-8" + ) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + node_ids = {n["id"] for n in data["nodes"]} + assert {"auth_token_validation", "login_session_verification"} <= node_ids + + data["links"].extend([ + { + "source": "auth_token_validation", + "target": "login_session_verification", + "relation": "semantically_similar_to", + "confidence": "INFERRED", + "source_file": "auth.md", + }, + # A stale AST-tier edge of the same source must still be evicted. + { + "source": "auth_token_validation", + "target": "login_session_verification", + "relation": "references", + "_origin": "ast", + "source_file": "auth.md", + }, + ]) + graph_path.write_text(json.dumps(data), encoding="utf-8") + + assert _rebuild_code( + corpus, + changed_paths=changed_paths, + no_cluster=True, + acquire_lock=False, + ) is True + + after = json.loads(graph_path.read_text(encoding="utf-8")) + relations = { + (e.get("source"), e.get("target"), e.get("relation")) + for e in after["links"] + } + assert ( + "auth_token_validation", "login_session_verification", "semantically_similar_to" + ) in relations, "semantic edge from a re-extracted doc must survive an AST-only update" + assert ( + "auth_token_validation", "login_session_verification", "references" + ) not in relations, "stale AST-tier edge of a re-extracted source must be evicted" + + @pytest.mark.parametrize( "changed_paths", [None, [Path("only.py")]], @@ -641,9 +1077,9 @@ def test_watch_loads_graphifyignore_once(tmp_path, monkeypatch): calls = {"n": 0} real_loader = detect_mod._load_graphifyignore - def counting_loader(root): + def counting_loader(root, **kwargs): calls["n"] += 1 - return real_loader(root) + return real_loader(root, **kwargs) # Patch the symbol the watch module imported at module-load time. monkeypatch.setattr(watch_mod, "_load_graphifyignore", counting_loader) @@ -1309,3 +1745,539 @@ def test_merge_changed_paths_dedupes_in_order(): [Path("a.py")], ) assert [p.as_posix() for p in merged] == ["a.py", "b.py", "c.py"] + + +def test_rebuild_code_preserves_nodes_from_excluded_but_alive_file(tmp_path, capsys): + """Fail-closed eviction: a file that leaves the scan corpus (newly ignored) + but still exists on disk was EXCLUDED, not deleted — its nodes must survive + an incremental rebuild, with a loud message, instead of being silently + mass-evicted as stale sources (the docs/brainstorms incident: an upgrade + started honoring .gitignore and evicted 655 nodes whose files were present). + """ + import json + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + (corpus / "notes").mkdir(parents=True) + (corpus / "auth.py").write_text("def login(): pass\n", encoding="utf-8") + (corpus / "notes" / "brainstorm.md").write_text( + "# Brainstorm\n\nA local-only design note.\n", encoding="utf-8" + ) + + assert _rebuild_code(corpus, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + labels = {n["label"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + assert "brainstorm.md" in labels + + # The file becomes ignored (leaves the corpus) but stays on disk. + (corpus / ".graphifyignore").write_text("notes/\n", encoding="utf-8") + capsys.readouterr() + + assert _rebuild_code(corpus, changed_paths=[Path("auth.py")], acquire_lock=False) is True + labels = {n["label"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + assert "brainstorm.md" in labels, ( + "nodes from an excluded-but-alive file must be preserved, not evicted" + ) + assert "fail-closed: kept" in capsys.readouterr().out + + +def test_rebuild_code_still_evicts_when_excluded_file_is_also_deleted(tmp_path): + """The fail-closed preserve must not weaken true-deletion eviction: once the + excluded file is actually gone from disk, its nodes are evicted as before.""" + import json + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + (corpus / "notes").mkdir(parents=True) + (corpus / "auth.py").write_text("def login(): pass\n", encoding="utf-8") + (corpus / "notes" / "brainstorm.md").write_text("# Brainstorm\n", encoding="utf-8") + + assert _rebuild_code(corpus, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + + (corpus / "notes" / "brainstorm.md").unlink() + + assert _rebuild_code(corpus, changed_paths=[Path("auth.py")], acquire_lock=False) is True + labels = {n["label"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + assert "brainstorm.md" not in labels, "deleted file's nodes must still be evicted" + assert "login()" in labels + + +# --- #1915: semantic-backed docs must not be double-represented by the AST quick-scan --- + + +_SEMANTIC_GUIDE_IDS = {"guide_doc", "auth_flow", "session_model"} +_AST_GUIDE_IDS = {"guide", "guide_overview", "guide_setup", "guide_usage"} + + +def _seed_semantic_doc_graph(corpus): + """Build a code-only graph, then add guide.md represented ONLY semantically. + + Mimics a graph produced by the CLI ``graphify . --update`` path: code AST + nodes plus a semantic (LLM) layer for the document — a ``_doc`` node + and concept nodes, none carrying the ``_origin`` marker — and NO AST + heading nodes for the doc. + """ + from graphify.watch import _rebuild_code + + corpus.mkdir() + (corpus / "app.py").write_text( + "def handle_login():\n return 1\n", encoding="utf-8" + ) + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + (corpus / "guide.md").write_text( + "# Overview\n\nIntro.\n\n## Setup\n\nSteps.\n\n## Usage\n\nMore.\n", + encoding="utf-8", + ) + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + code_node_id = next( + n["id"] for n in data["nodes"] if n.get("source_file") == "app.py" + ) + data["nodes"].extend([ + {"id": "guide_doc", "label": "Guide", "file_type": "document", + "source_file": "guide.md"}, + {"id": "auth_flow", "label": "Auth Flow", "file_type": "concept", + "source_file": "guide.md"}, + {"id": "session_model", "label": "Session Model", "file_type": "concept", + "source_file": "guide.md"}, + ]) + data["links"].extend([ + {"source": "guide_doc", "target": "auth_flow", "relation": "explains", + "confidence": "INFERRED", "source_file": "guide.md"}, + {"source": "auth_flow", "target": code_node_id, + "relation": "implemented_by", "confidence": "INFERRED", + "source_file": "guide.md"}, + ]) + graph_path.write_text(json.dumps(data), encoding="utf-8") + return graph_path + + +_CONCEPT_ONLY_GUIDE_IDS = {"auth_flow", "session_model"} + + +def _seed_semantic_doc_graph_concept_only(corpus): + """Like ``_seed_semantic_doc_graph``, but guide.md's semantic layer is + ONLY concept/rationale nodes (no ``file_type=="document"`` node) — the + extraction spec's preferred shape for a doc full of named concepts (#1954). + """ + from graphify.watch import _rebuild_code + + corpus.mkdir() + (corpus / "app.py").write_text( + "def handle_login():\n return 1\n", encoding="utf-8" + ) + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + (corpus / "guide.md").write_text( + "# Overview\n\nIntro.\n\n## Setup\n\nSteps.\n\n## Usage\n\nMore.\n", + encoding="utf-8", + ) + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + code_node_id = next( + n["id"] for n in data["nodes"] if n.get("source_file") == "app.py" + ) + data["nodes"].extend([ + {"id": "auth_flow", "label": "Auth Flow", "file_type": "concept", + "source_file": "guide.md"}, + {"id": "session_model", "label": "Session Model", "file_type": "rationale", + "source_file": "guide.md"}, + ]) + data["links"].extend([ + {"source": "auth_flow", "target": "session_model", "relation": "explains", + "confidence": "INFERRED", "source_file": "guide.md"}, + {"source": "auth_flow", "target": code_node_id, + "relation": "implemented_by", "confidence": "INFERRED", + "source_file": "guide.md"}, + ]) + graph_path.write_text(json.dumps(data), encoding="utf-8") + return graph_path + + +def test_rebuild_code_semantic_doc_not_double_represented_on_full_rebuild(tmp_path): + """#1915: a full _rebuild_code must not AST-quick-scan a doc whose semantic + (LLM) nodes already represent it. Before the fix the quick-scan minted + heading nodes ON TOP of the preserved semantic nodes, representing every + doc twice (~4x bloated graph vs the CLI update path).""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + graph_path = _seed_semantic_doc_graph(corpus) + before = json.loads(graph_path.read_text(encoding="utf-8")) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + after = json.loads(graph_path.read_text(encoding="utf-8")) + after_ids = {n["id"] for n in after["nodes"]} + assert _SEMANTIC_GUIDE_IDS <= after_ids, "semantic doc nodes must be preserved" + assert not (_AST_GUIDE_IDS & after_ids), ( + "AST heading nodes minted for a semantic-backed doc (#1915)" + ) + assert len(after["nodes"]) == len(before["nodes"]), ( + f"node count inflated {len(before['nodes'])} -> {len(after['nodes'])} (#1915)" + ) + + +def test_rebuild_code_concept_only_semantic_doc_not_double_represented_on_full_rebuild( + tmp_path, +): + """#1954: a doc represented ONLY by concept/rationale nodes (no + file_type=="document" node) must also be recognized as semantic-backed + and skipped by the AST quick-scan — not just docs with a "document" node.""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + graph_path = _seed_semantic_doc_graph_concept_only(corpus) + before = json.loads(graph_path.read_text(encoding="utf-8")) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + after = json.loads(graph_path.read_text(encoding="utf-8")) + after_ids = {n["id"] for n in after["nodes"]} + assert _CONCEPT_ONLY_GUIDE_IDS <= after_ids, "semantic doc nodes must be preserved" + assert not (_AST_GUIDE_IDS & after_ids), ( + "AST heading nodes minted for a concept-only semantic-backed doc (#1954)" + ) + assert len(after["nodes"]) == len(before["nodes"]), ( + f"node count inflated {len(before['nodes'])} -> {len(after['nodes'])} (#1954)" + ) + + +@pytest.mark.parametrize( + "changed", + [[Path("guide.md")], [Path("guide.md"), Path("app.py")]], + ids=["doc-only", "doc-plus-code"], +) +def test_rebuild_code_incremental_preserves_semantic_doc_nodes_and_edges( + tmp_path, changed +): + """#1915: an incremental rebuild whose change set includes a semantic-backed + doc must not wipe the doc's semantic nodes or their edges — re-extraction + owns only a source's AST tier (node-level mirror of #1865's edge rule).""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + graph_path = _seed_semantic_doc_graph(corpus) + + assert _rebuild_code( + corpus, changed_paths=changed, no_cluster=True, acquire_lock=False + ) is True + + after = json.loads(graph_path.read_text(encoding="utf-8")) + after_ids = {n["id"] for n in after["nodes"]} + assert _SEMANTIC_GUIDE_IDS <= after_ids, ( + "semantic doc nodes wiped by an incremental rebuild" + ) + relations = { + (e.get("source"), e.get("target"), e.get("relation")) + for e in after["links"] + } + assert ("guide_doc", "auth_flow", "explains") in relations, ( + "semantic doc edge dropped by an incremental rebuild" + ) + assert any( + src == "auth_flow" and rel == "implemented_by" + for src, _tgt, rel in relations + ), "doc-to-code semantic edge dropped by an incremental rebuild" + assert not (_AST_GUIDE_IDS & after_ids), ( + "incremental rebuild AST-quick-scanned a semantic-backed doc (#1915)" + ) + + +@pytest.mark.parametrize( + "changed", + [[Path("guide.md")], [Path("guide.md"), Path("app.py")]], + ids=["doc-only", "doc-plus-code"], +) +def test_rebuild_code_incremental_preserves_concept_only_semantic_doc_nodes_and_edges( + tmp_path, changed +): + """#1954: incremental analogue — a concept/rationale-only semantic doc + must not lose its nodes/edges nor get AST-quick-scanned on an incremental + rebuild, mirroring the #1915 doc-node case above.""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + graph_path = _seed_semantic_doc_graph_concept_only(corpus) + + assert _rebuild_code( + corpus, changed_paths=changed, no_cluster=True, acquire_lock=False + ) is True + + after = json.loads(graph_path.read_text(encoding="utf-8")) + after_ids = {n["id"] for n in after["nodes"]} + assert _CONCEPT_ONLY_GUIDE_IDS <= after_ids, ( + "concept-only semantic doc nodes wiped by an incremental rebuild" + ) + relations = { + (e.get("source"), e.get("target"), e.get("relation")) + for e in after["links"] + } + assert ("auth_flow", "session_model", "explains") in relations, ( + "concept-only semantic doc edge dropped by an incremental rebuild" + ) + assert any( + src == "auth_flow" and rel == "implemented_by" + for src, _tgt, rel in relations + ), "doc-to-code semantic edge dropped by an incremental rebuild" + assert not (_AST_GUIDE_IDS & after_ids), ( + "incremental rebuild AST-quick-scanned a concept-only semantic-backed doc (#1954)" + ) + + +def test_rebuild_code_quick_scans_doc_without_semantic_nodes(tmp_path): + """#09b33b7 guard: a doc with NO semantic layer still gets the AST + quick-scan so no-LLM corpora keep their heading structure — #1915's + semantic-supersedes-AST rule must not regress the fallback.""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "app.py").write_text("def f():\n return 1\n", encoding="utf-8") + (corpus / "notes.md").write_text("# Alpha\n\n## Beta\n", encoding="utf-8") + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + assert {"notes", "notes_alpha", "notes_beta"} <= ids + + # A rebuild over the existing graph (still no semantic nodes for the doc) + # keeps quick-scanning it rather than dropping its structure. + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + assert {"notes", "notes_alpha", "notes_beta"} <= ids + + +def test_rebuild_code_polluted_graph_self_heals_on_full_rebuild(tmp_path): + """#1915: a graph already bloated by the bug (semantic doc nodes PLUS stale + _origin=="ast" heading nodes for the same doc) sheds the heading nodes on + the next full rebuild via the AST ownership rule — and the shrink guard + accepts the smaller write without --force.""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "app.py").write_text( + "def handle_login():\n return 1\n", encoding="utf-8" + ) + (corpus / "guide.md").write_text( + "# Overview\n\n## Setup\n\n## Usage\n", encoding="utf-8" + ) + # Initial build quick-scans guide.md (no semantic layer yet): AST nodes. + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + assert _AST_GUIDE_IDS <= {n["id"] for n in data["nodes"]} + + # Layer the semantic representation on top -> the double-represented state. + data["nodes"].extend([ + {"id": "guide_doc", "label": "Guide", "file_type": "document", + "source_file": "guide.md"}, + {"id": "auth_flow", "label": "Auth Flow", "file_type": "concept", + "source_file": "guide.md"}, + ]) + data["links"].append({ + "source": "guide_doc", "target": "auth_flow", "relation": "explains", + "confidence": "INFERRED", "source_file": "guide.md", + }) + graph_path.write_text(json.dumps(data), encoding="utf-8") + nodes_before = len(data["nodes"]) + + # No force=True: the self-heal shrink must be accepted by the guard. + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + after = json.loads(graph_path.read_text(encoding="utf-8")) + after_ids = {n["id"] for n in after["nodes"]} + assert {"guide_doc", "auth_flow"} <= after_ids + assert not (_AST_GUIDE_IDS & after_ids), ( + "stale AST heading nodes for a semantic-backed doc must self-heal away" + ) + assert len(after["nodes"]) < nodes_before, "polluted graph should shrink" + + +# ── #2014: code-typed semantic nodes count as a doc's semantic layer ─────────── + +_CODE_ONLY_GUIDE_IDS = {"parse_config", "load_settings"} + + +def _seed_semantic_doc_graph_code_only(corpus): + """Like ``_seed_semantic_doc_graph``, but guide.md's semantic layer is ONLY + code-typed nodes — symbols the LLM surfaced from WITHIN the doc (llm.py + ``_bind_node_evidence``), with no document/concept node at all (#2014).""" + from graphify.watch import _rebuild_code + + corpus.mkdir() + (corpus / "app.py").write_text( + "def handle_login():\n return 1\n", encoding="utf-8" + ) + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + (corpus / "guide.md").write_text( + "# Overview\n\nIntro.\n\n## Setup\n\nSteps.\n\n## Usage\n\nMore.\n", + encoding="utf-8", + ) + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + code_node_id = next( + n["id"] for n in data["nodes"] if n.get("source_file") == "app.py" + ) + data["nodes"].extend([ + {"id": "parse_config", "label": "parse_config()", "file_type": "code", + "source_file": "guide.md"}, + {"id": "load_settings", "label": "load_settings()", "file_type": "code", + "source_file": "guide.md"}, + ]) + data["links"].append({ + "source": "parse_config", "target": code_node_id, + "relation": "implemented_by", "confidence": "INFERRED", + "source_file": "guide.md", + }) + graph_path.write_text(json.dumps(data), encoding="utf-8") + return graph_path + + +def test_rebuild_code_code_only_semantic_doc_not_double_represented_on_full_rebuild( + tmp_path, +): + """#2014: a doc represented ONLY by code-typed semantic nodes (symbols + surfaced from within it) must be recognized as semantic-backed and skipped + by the AST quick-scan. Before the fix "code" was absent from the semantic + file_type gate, so the doc was re-AST-scanned — minting heading nodes AND + dropping the code-typed semantic nodes (they belonged to a now-rebuilt + source), silently losing them.""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + graph_path = _seed_semantic_doc_graph_code_only(corpus) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + + after = json.loads(graph_path.read_text(encoding="utf-8")) + after_ids = {n["id"] for n in after["nodes"]} + assert _CODE_ONLY_GUIDE_IDS <= after_ids, ( + "code-typed semantic doc nodes dropped by a full rebuild (#2014)" + ) + assert not (_AST_GUIDE_IDS & after_ids), ( + "AST heading nodes minted for a code-only semantic-backed doc (#2014)" + ) + + +# ── #2051: deleted non-AST sources (docs/papers/images) get evicted ──────────── + +def test_rebuild_code_evicts_semantic_nodes_from_deleted_non_ast_source(tmp_path): + """#2051: a full `graphify update` must evict semantic nodes whose non-AST + source file (a .txt/.pdf/.png with no code extractor) was deleted from disk. + The corpus sweep used to skip every sourceless-of-extractor node, so those + nodes survived forever and were served as authoritative long after the file + was gone. Disk absence is the only deletion evidence for such sources.""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "app.py").write_text("def handle():\n return 1\n", encoding="utf-8") + # Two non-AST semantic sources: one stays on disk, one gets deleted. + (corpus / "kept.txt").write_text("Design rationale that stays.\n", encoding="utf-8") + (corpus / "gone.txt").write_text("Rationale that will be deleted.\n", encoding="utf-8") + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + # No LLM in tests, so inject the semantic layer these .txt files would carry. + data["nodes"].extend([ + {"id": "kept_concept", "label": "Kept Concept", "file_type": "concept", + "source_file": "kept.txt"}, + {"id": "gone_concept", "label": "Gone Concept", "file_type": "concept", + "source_file": "gone.txt"}, + ]) + graph_path.write_text(json.dumps(data), encoding="utf-8") + + # Delete one non-AST source; the other stays. + (corpus / "gone.txt").unlink() + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + after_ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + assert "gone_concept" not in after_ids, ( + "semantic node from a deleted non-AST source must be evicted (#2051)" + ) + assert "kept_concept" in after_ids, ( + "semantic node from a surviving non-AST source must be preserved" + ) + + +def test_rebuild_code_preserves_remote_source_across_repeated_updates(tmp_path): + """#2051 follow-up: a node whose source_file is a URL/virtual scheme + (gdoc://, s3://, http://) must survive REPEATED `graphify update`s. Path + normalization on the write side collapses the double slash (`gdoc://x` -> + `gdoc:/x`), so a literal `"://"` guard matched on the first update but missed + on the second, dropping the node into the disk-absence eviction branch + (Path('gdoc:/x').exists() is False) — a data-loss regression from the #2051 + disk-absence sweep. The scheme is now matched with a regex tolerant of the + collapse.""" + from graphify.watch import _rebuild_code, _is_remote_source + + # unit-level: the guard tolerates the slash collapse and rejects local paths + assert _is_remote_source("gdoc://abc") + assert _is_remote_source("gdoc:/abc") # collapsed form + assert _is_remote_source("s3://bucket/key") + assert _is_remote_source("https://example.com/doc") + assert not _is_remote_source("src/app.py") + assert not _is_remote_source("notes.txt") + assert not _is_remote_source("C:/Users/x/a.py") # Windows drive != scheme + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "app.py").write_text("def handle():\n return 1\n", encoding="utf-8") + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + data["nodes"].append( + {"id": "remote_doc", "label": "Remote Spec", "file_type": "document", + "source_file": "gdoc://team/spec"} + ) + graph_path.write_text(json.dumps(data), encoding="utf-8") + + # Three consecutive full updates: the remote node must persist through every + # one, even after its stored source_file is normalized to the collapsed form. + for i in range(3): + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + after = json.loads(graph_path.read_text(encoding="utf-8")) + ids = {n["id"] for n in after["nodes"]} + assert "remote_doc" in ids, f"remote-source node evicted on update #{i + 1} (#2051 follow-up)" + + +# ── #2056: present-but-unextractable files in a change set are not deletions ─── + +def test_rebuild_code_incremental_preserves_present_non_ast_source(tmp_path): + """#2056: an incremental rebuild whose change set names a file that exists but + has no AST extractor (a doc/paper/image, or an excluded path) must NOT treat + it as deleted. The old change-set loop routed any present-but-untracked file + to _add_deleted_source, evicting its semantic nodes AND flipping + had_explicit_deletions so the shrink guard waved the loss through.""" + from graphify.watch import _rebuild_code + + corpus = tmp_path / "corpus" + corpus.mkdir() + (corpus / "app.py").write_text("def handle():\n return 1\n", encoding="utf-8") + (corpus / "spec.txt").write_text("A spec with a semantic layer.\n", encoding="utf-8") + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + graph_path = corpus / "graphify-out" / "graph.json" + data = json.loads(graph_path.read_text(encoding="utf-8")) + data["nodes"].append( + {"id": "spec_concept", "label": "Spec Concept", "file_type": "concept", + "source_file": "spec.txt"} + ) + graph_path.write_text(json.dumps(data), encoding="utf-8") + + # spec.txt is present but not AST-extractable; app.py is a real code change. + assert _rebuild_code( + corpus, changed_paths=[Path("spec.txt"), Path("app.py")], + no_cluster=True, acquire_lock=False, + ) is True + + after_ids = {n["id"] for n in json.loads(graph_path.read_text(encoding="utf-8"))["nodes"]} + assert "spec_concept" in after_ids, ( + "present-but-unextractable file in change set wrongly evicted as deleted (#2056)" + ) diff --git a/tests/test_word_count_cache.py b/tests/test_word_count_cache.py index 75bbb8392..cfa5f5a35 100644 --- a/tests/test_word_count_cache.py +++ b/tests/test_word_count_cache.py @@ -49,4 +49,53 @@ def test_word_count_augments_existing_hash_entry(tmp_path, monkeypatch): assert cache.file_hash(f, tmp_path) == h key = str(cache._normalize_path(f).resolve()) entry = cache._stat_index[key] - assert entry.get("hash") == h and entry.get("word_count") == 3 + # #1989: digests are now stored per salt under "hashes" (salt = path relative + # to root == "m.py" here), co-located with the word_count. + assert entry.get("hashes", {}).get("m.py") == h and entry.get("word_count") == 3 + + +def test_file_hash_is_order_independent_across_roots(tmp_path, monkeypatch): + """#1989: the stat-index memo must be keyed by the salt (path relative to + root) that enters the digest, so the same (file, root) returns the same + digest regardless of what root was hashed first.""" + import hashlib + from graphify import cache + monkeypatch.setattr(cache, "_stat_index", {}) + monkeypatch.setattr(cache, "_stat_index_root", None) + + root_a = tmp_path / "a"; root_a.mkdir() + f = root_a / "doc.txt"; f.write_text("hello world\n") + root_b = tmp_path / "b"; root_b.mkdir() # f is NOT under root_b -> abs-path salt + + content = f.read_bytes() + exp_rel = hashlib.sha256(content + b"\x00" + b"doc.txt").hexdigest() + exp_abs = hashlib.sha256( + content + b"\x00" + str(cache._normalize_path(f).resolve()).replace("\\", "/").lower().encode() + ).hexdigest() + + # rel-first order + assert cache.file_hash(f, root_a) == exp_rel + assert cache.file_hash(f, root_b) == exp_abs # not served the rel digest + assert cache.file_hash(f, root_a) == exp_rel # still stable + + # abs-first order, fresh index + monkeypatch.setattr(cache, "_stat_index", {}) + monkeypatch.setattr(cache, "_stat_index_root", None) + assert cache.file_hash(f, root_b) == exp_abs + assert cache.file_hash(f, root_a) == exp_rel # not served the abs digest + + +def test_file_hash_ignores_legacy_unsalted_entry(tmp_path, monkeypatch): + """A pre-#1989 entry carrying a bare "hash" (no salt) is never trusted.""" + import hashlib + from graphify import cache + monkeypatch.setattr(cache, "_stat_index", {}) + monkeypatch.setattr(cache, "_stat_index_root", None) + f = tmp_path / "m.py"; f.write_text("x = 1\n") + st = f.stat() + key = str(cache._normalize_path(f).resolve()) + cache._stat_index[key] = {"size": st.st_size, "mtime_ns": st.st_mtime_ns, "hash": "deadbeef"} + exp = hashlib.sha256(f.read_bytes() + b"\x00" + b"m.py").hexdigest() + assert cache.file_hash(f, tmp_path) == exp # recomputed, not "deadbeef" + entry = cache._stat_index[key] + assert "hash" not in entry and entry["hashes"]["m.py"] == exp diff --git a/tools/skillgen/expected/graphify__skill-agents.md b/tools/skillgen/expected/graphify__skill-agents.md index 2e19931fa..afb4ecc12 100644 --- a/tools/skillgen/expected/graphify__skill-agents.md +++ b/tools/skillgen/expected/graphify__skill-agents.md @@ -10,7 +10,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## Usage ``` -/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) /graphify # full pipeline on specific path /graphify https://github.com// # clone repo then run full pipeline on it /graphify https://github.com// --branch # clone a specific branch @@ -70,7 +70,7 @@ PYTHON="" GRAPHIFY_BIN=$(which graphify 2>/dev/null) # 1. uv tool installs — most reliable on modern Mac/Linux if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi fi # 2. Read shebang from graphify binary (pipx and direct pip installs) @@ -86,7 +86,7 @@ if [ -z "$PYTHON" ]; then PYTHON="python3"; fi if ! "$PYTHON" -c "import graphify" 2>/dev/null; then if command -v uv >/dev/null 2>&1; then uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi else "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ @@ -131,7 +131,7 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). @@ -214,6 +214,8 @@ Before dispatching subagents, print a timing estimate: Before dispatching any subagents, check which files already have cached extraction results: +SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. + ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -226,7 +228,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -302,7 +304,7 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' " ``` -Save new results to cache: +Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -310,7 +312,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') " ``` @@ -548,15 +551,37 @@ from graphify.detect import save_manifest # Save manifest for --update detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) # In --update mode, 'all_files' carries the full corpus; 'files' is the changed # subset. Full-rebuild mode populates only 'files', so the fallback handles that. # root= relativizes the manifest keys to the scan root (same base as the build), # so the on-disk manifest is portable across clones/machines and a later --update # matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: +# a detected file whose chunk failed or was omitted must stay unstamped so the +# next --update re-queues it, otherwise it is marked done and its content is lost +# forever (#2015). This mirrors the library extract path exactly +# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the +# raw corpus. Code files are always stamped (AST is deterministic); only semantic +# types are gated on output. +from graphify.cli import _stamped_manifest_files +_corpus = detect.get('all_files') or detect['files'] +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +# Files dispatched this run (the changed subset) but NOT stamped above still carry +# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues +# them instead of reading them as unchanged (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root +# files newly excluded since last run are dropped rather than masquerading as +# deletions; untouched files' prior rows are still preserved (#1908). +_scan = {f for fl in _corpus.values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) diff --git a/tools/skillgen/expected/graphify__skill-aider.md b/tools/skillgen/expected/graphify__skill-aider.md index 2ccf5326c..4f03ccbae 100644 --- a/tools/skillgen/expected/graphify__skill-aider.md +++ b/tools/skillgen/expected/graphify__skill-aider.md @@ -10,7 +10,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## Usage ``` -/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) /graphify # full pipeline on specific path /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files @@ -64,7 +64,7 @@ PYTHON="" GRAPHIFY_BIN=$(which graphify 2>/dev/null) # 1. uv tool installs — most reliable on modern Mac/Linux if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi fi # 2. Read shebang from graphify binary (pipx and direct pip installs) @@ -80,7 +80,7 @@ if [ -z "$PYTHON" ]; then PYTHON="python3"; fi if ! "$PYTHON" -c "import graphify" 2>/dev/null; then if command -v uv >/dev/null 2>&1; then uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi else "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ @@ -123,7 +123,7 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. - Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +uncached = [line for line in Path('.graphify_uncached.txt').read_text().splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached) print(f'Cached {saved} files') " ``` @@ -676,10 +677,19 @@ from graphify.detect import save_manifest # Save manifest for --update detect = json.loads(Path('.graphify_detect.json').read_text()) -save_manifest(detect['files'], root='INPUT_PATH') +extract = json.loads(Path('.graphify_extract.json').read_text()) +# Stamp only semantic files that produced output so a failed chunk is re-queued next run, not lost (#2015). +from graphify.cli import _stamped_manifest_files +_corpus = detect.get('all_files') or detect['files'] +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +_scan = {f for fl in _corpus.values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker -extract = json.loads(Path('.graphify_extract.json').read_text()) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) diff --git a/tools/skillgen/expected/graphify__skill-amp.md b/tools/skillgen/expected/graphify__skill-amp.md index 2e19931fa..afb4ecc12 100644 --- a/tools/skillgen/expected/graphify__skill-amp.md +++ b/tools/skillgen/expected/graphify__skill-amp.md @@ -10,7 +10,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## Usage ``` -/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) /graphify # full pipeline on specific path /graphify https://github.com// # clone repo then run full pipeline on it /graphify https://github.com// --branch # clone a specific branch @@ -70,7 +70,7 @@ PYTHON="" GRAPHIFY_BIN=$(which graphify 2>/dev/null) # 1. uv tool installs — most reliable on modern Mac/Linux if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi fi # 2. Read shebang from graphify binary (pipx and direct pip installs) @@ -86,7 +86,7 @@ if [ -z "$PYTHON" ]; then PYTHON="python3"; fi if ! "$PYTHON" -c "import graphify" 2>/dev/null; then if command -v uv >/dev/null 2>&1; then uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi else "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ @@ -131,7 +131,7 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). @@ -214,6 +214,8 @@ Before dispatching subagents, print a timing estimate: Before dispatching any subagents, check which files already have cached extraction results: +SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. + ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -226,7 +228,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -302,7 +304,7 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' " ``` -Save new results to cache: +Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -310,7 +312,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') " ``` @@ -548,15 +551,37 @@ from graphify.detect import save_manifest # Save manifest for --update detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) # In --update mode, 'all_files' carries the full corpus; 'files' is the changed # subset. Full-rebuild mode populates only 'files', so the fallback handles that. # root= relativizes the manifest keys to the scan root (same base as the build), # so the on-disk manifest is portable across clones/machines and a later --update # matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: +# a detected file whose chunk failed or was omitted must stay unstamped so the +# next --update re-queues it, otherwise it is marked done and its content is lost +# forever (#2015). This mirrors the library extract path exactly +# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the +# raw corpus. Code files are always stamped (AST is deterministic); only semantic +# types are gated on output. +from graphify.cli import _stamped_manifest_files +_corpus = detect.get('all_files') or detect['files'] +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +# Files dispatched this run (the changed subset) but NOT stamped above still carry +# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues +# them instead of reading them as unchanged (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root +# files newly excluded since last run are dropped rather than masquerading as +# deletions; untouched files' prior rows are still preserved (#1908). +_scan = {f for fl in _corpus.values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) diff --git a/tools/skillgen/expected/graphify__skill-claw.md b/tools/skillgen/expected/graphify__skill-claw.md index b3542438d..d98865cc8 100644 --- a/tools/skillgen/expected/graphify__skill-claw.md +++ b/tools/skillgen/expected/graphify__skill-claw.md @@ -10,7 +10,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## Usage ``` -/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) /graphify # full pipeline on specific path /graphify https://github.com// # clone repo then run full pipeline on it /graphify https://github.com// --branch # clone a specific branch @@ -70,7 +70,7 @@ PYTHON="" GRAPHIFY_BIN=$(which graphify 2>/dev/null) # 1. uv tool installs — most reliable on modern Mac/Linux if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi fi # 2. Read shebang from graphify binary (pipx and direct pip installs) @@ -86,7 +86,7 @@ if [ -z "$PYTHON" ]; then PYTHON="python3"; fi if ! "$PYTHON" -c "import graphify" 2>/dev/null; then if command -v uv >/dev/null 2>&1; then uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi else "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ @@ -131,7 +131,7 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). @@ -214,6 +214,8 @@ Before dispatching subagents, print a timing estimate: Before dispatching any subagents, check which files already have cached extraction results: +SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. + ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -226,7 +228,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -305,7 +307,7 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' " ``` -Save new results to cache: +Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -313,7 +315,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') " ``` @@ -551,15 +554,37 @@ from graphify.detect import save_manifest # Save manifest for --update detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) # In --update mode, 'all_files' carries the full corpus; 'files' is the changed # subset. Full-rebuild mode populates only 'files', so the fallback handles that. # root= relativizes the manifest keys to the scan root (same base as the build), # so the on-disk manifest is portable across clones/machines and a later --update # matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: +# a detected file whose chunk failed or was omitted must stay unstamped so the +# next --update re-queues it, otherwise it is marked done and its content is lost +# forever (#2015). This mirrors the library extract path exactly +# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the +# raw corpus. Code files are always stamped (AST is deterministic); only semantic +# types are gated on output. +from graphify.cli import _stamped_manifest_files +_corpus = detect.get('all_files') or detect['files'] +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +# Files dispatched this run (the changed subset) but NOT stamped above still carry +# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues +# them instead of reading them as unchanged (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root +# files newly excluded since last run are dropped rather than masquerading as +# deletions; untouched files' prior rows are still preserved (#1908). +_scan = {f for fl in _corpus.values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) diff --git a/tools/skillgen/expected/graphify__skill-codex.md b/tools/skillgen/expected/graphify__skill-codex.md index e6e411ea0..0c821a278 100644 --- a/tools/skillgen/expected/graphify__skill-codex.md +++ b/tools/skillgen/expected/graphify__skill-codex.md @@ -10,7 +10,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## Usage ``` -/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) /graphify # full pipeline on specific path /graphify https://github.com// # clone repo then run full pipeline on it /graphify https://github.com// --branch # clone a specific branch @@ -70,7 +70,7 @@ PYTHON="" GRAPHIFY_BIN=$(which graphify 2>/dev/null) # 1. uv tool installs — most reliable on modern Mac/Linux if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi fi # 2. Read shebang from graphify binary (pipx and direct pip installs) @@ -86,7 +86,7 @@ if [ -z "$PYTHON" ]; then PYTHON="python3"; fi if ! "$PYTHON" -c "import graphify" 2>/dev/null; then if command -v uv >/dev/null 2>&1; then uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi else "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ @@ -131,7 +131,7 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). @@ -214,6 +214,8 @@ Before dispatching subagents, print a timing estimate: Before dispatching any subagents, check which files already have cached extraction results: +SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. + ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -226,7 +228,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -302,7 +304,7 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' " ``` -Save new results to cache: +Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -310,7 +312,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') " ``` @@ -548,15 +551,37 @@ from graphify.detect import save_manifest # Save manifest for --update detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) # In --update mode, 'all_files' carries the full corpus; 'files' is the changed # subset. Full-rebuild mode populates only 'files', so the fallback handles that. # root= relativizes the manifest keys to the scan root (same base as the build), # so the on-disk manifest is portable across clones/machines and a later --update # matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: +# a detected file whose chunk failed or was omitted must stay unstamped so the +# next --update re-queues it, otherwise it is marked done and its content is lost +# forever (#2015). This mirrors the library extract path exactly +# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the +# raw corpus. Code files are always stamped (AST is deterministic); only semantic +# types are gated on output. +from graphify.cli import _stamped_manifest_files +_corpus = detect.get('all_files') or detect['files'] +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +# Files dispatched this run (the changed subset) but NOT stamped above still carry +# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues +# them instead of reading them as unchanged (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root +# files newly excluded since last run are dropped rather than masquerading as +# deletions; untouched files' prior rows are still preserved (#1908). +_scan = {f for fl in _corpus.values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) diff --git a/tools/skillgen/expected/graphify__skill-copilot.md b/tools/skillgen/expected/graphify__skill-copilot.md index b3542438d..d98865cc8 100644 --- a/tools/skillgen/expected/graphify__skill-copilot.md +++ b/tools/skillgen/expected/graphify__skill-copilot.md @@ -10,7 +10,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## Usage ``` -/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) /graphify # full pipeline on specific path /graphify https://github.com// # clone repo then run full pipeline on it /graphify https://github.com// --branch # clone a specific branch @@ -70,7 +70,7 @@ PYTHON="" GRAPHIFY_BIN=$(which graphify 2>/dev/null) # 1. uv tool installs — most reliable on modern Mac/Linux if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi fi # 2. Read shebang from graphify binary (pipx and direct pip installs) @@ -86,7 +86,7 @@ if [ -z "$PYTHON" ]; then PYTHON="python3"; fi if ! "$PYTHON" -c "import graphify" 2>/dev/null; then if command -v uv >/dev/null 2>&1; then uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi else "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ @@ -131,7 +131,7 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). @@ -214,6 +214,8 @@ Before dispatching subagents, print a timing estimate: Before dispatching any subagents, check which files already have cached extraction results: +SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. + ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -226,7 +228,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -305,7 +307,7 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' " ``` -Save new results to cache: +Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -313,7 +315,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') " ``` @@ -551,15 +554,37 @@ from graphify.detect import save_manifest # Save manifest for --update detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) # In --update mode, 'all_files' carries the full corpus; 'files' is the changed # subset. Full-rebuild mode populates only 'files', so the fallback handles that. # root= relativizes the manifest keys to the scan root (same base as the build), # so the on-disk manifest is portable across clones/machines and a later --update # matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: +# a detected file whose chunk failed or was omitted must stay unstamped so the +# next --update re-queues it, otherwise it is marked done and its content is lost +# forever (#2015). This mirrors the library extract path exactly +# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the +# raw corpus. Code files are always stamped (AST is deterministic); only semantic +# types are gated on output. +from graphify.cli import _stamped_manifest_files +_corpus = detect.get('all_files') or detect['files'] +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +# Files dispatched this run (the changed subset) but NOT stamped above still carry +# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues +# them instead of reading them as unchanged (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root +# files newly excluded since last run are dropped rather than masquerading as +# deletions; untouched files' prior rows are still preserved (#1908). +_scan = {f for fl in _corpus.values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) diff --git a/tools/skillgen/expected/graphify__skill-devin.md b/tools/skillgen/expected/graphify__skill-devin.md index 13fb37c85..e3e6d2dec 100644 --- a/tools/skillgen/expected/graphify__skill-devin.md +++ b/tools/skillgen/expected/graphify__skill-devin.md @@ -75,7 +75,7 @@ PYTHON="" GRAPHIFY_BIN=$(which graphify 2>/dev/null) # 1. uv tool installs — most reliable on modern Mac/Linux if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi fi # 2. Read shebang from graphify binary (pipx and direct pip installs) @@ -91,7 +91,7 @@ if [ -z "$PYTHON" ]; then PYTHON="python3"; fi if ! "$PYTHON" -c "import graphify" 2>/dev/null; then if command -v uv >/dev/null 2>&1; then uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi else "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ @@ -136,7 +136,7 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. - Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. @@ -374,7 +374,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text().splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached) print(f'Cached {saved} files') " ``` @@ -794,10 +795,19 @@ from graphify.detect import save_manifest # Save manifest for --update detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -save_manifest(detect['files'], root='INPUT_PATH') +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +# Stamp only semantic files that produced output so a failed chunk is re-queued next run, not lost (#2015). +from graphify.cli import _stamped_manifest_files +_corpus = detect.get('all_files') or detect['files'] +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +_scan = {f for fl in _corpus.values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) diff --git a/tools/skillgen/expected/graphify__skill-droid.md b/tools/skillgen/expected/graphify__skill-droid.md index d1147b903..c3815d556 100644 --- a/tools/skillgen/expected/graphify__skill-droid.md +++ b/tools/skillgen/expected/graphify__skill-droid.md @@ -10,7 +10,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## Usage ``` -/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) /graphify # full pipeline on specific path /graphify https://github.com// # clone repo then run full pipeline on it /graphify https://github.com// --branch # clone a specific branch @@ -70,7 +70,7 @@ PYTHON="" GRAPHIFY_BIN=$(which graphify 2>/dev/null) # 1. uv tool installs — most reliable on modern Mac/Linux if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi fi # 2. Read shebang from graphify binary (pipx and direct pip installs) @@ -86,7 +86,7 @@ if [ -z "$PYTHON" ]; then PYTHON="python3"; fi if ! "$PYTHON" -c "import graphify" 2>/dev/null; then if command -v uv >/dev/null 2>&1; then uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi else "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ @@ -131,7 +131,7 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). @@ -214,6 +214,8 @@ Before dispatching subagents, print a timing estimate: Before dispatching any subagents, check which files already have cached extraction results: +SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. + ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -226,7 +228,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -302,7 +304,7 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' " ``` -Save new results to cache: +Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -310,7 +312,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') " ``` @@ -548,15 +551,37 @@ from graphify.detect import save_manifest # Save manifest for --update detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) # In --update mode, 'all_files' carries the full corpus; 'files' is the changed # subset. Full-rebuild mode populates only 'files', so the fallback handles that. # root= relativizes the manifest keys to the scan root (same base as the build), # so the on-disk manifest is portable across clones/machines and a later --update # matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: +# a detected file whose chunk failed or was omitted must stay unstamped so the +# next --update re-queues it, otherwise it is marked done and its content is lost +# forever (#2015). This mirrors the library extract path exactly +# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the +# raw corpus. Code files are always stamped (AST is deterministic); only semantic +# types are gated on output. +from graphify.cli import _stamped_manifest_files +_corpus = detect.get('all_files') or detect['files'] +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +# Files dispatched this run (the changed subset) but NOT stamped above still carry +# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues +# them instead of reading them as unchanged (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root +# files newly excluded since last run are dropped rather than masquerading as +# deletions; untouched files' prior rows are still preserved (#1908). +_scan = {f for fl in _corpus.values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) diff --git a/tools/skillgen/expected/graphify__skill-kilo.md b/tools/skillgen/expected/graphify__skill-kilo.md index c45578165..dbb4658ca 100644 --- a/tools/skillgen/expected/graphify__skill-kilo.md +++ b/tools/skillgen/expected/graphify__skill-kilo.md @@ -10,7 +10,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## Usage ``` -/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) /graphify # full pipeline on specific path /graphify https://github.com// # clone repo then run full pipeline on it /graphify https://github.com// --branch # clone a specific branch @@ -70,7 +70,7 @@ PYTHON="" GRAPHIFY_BIN=$(which graphify 2>/dev/null) # 1. uv tool installs — most reliable on modern Mac/Linux if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi fi # 2. Read shebang from graphify binary (pipx and direct pip installs) @@ -86,7 +86,7 @@ if [ -z "$PYTHON" ]; then PYTHON="python3"; fi if ! "$PYTHON" -c "import graphify" 2>/dev/null; then if command -v uv >/dev/null 2>&1; then uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi else "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ @@ -131,7 +131,7 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). @@ -214,6 +214,8 @@ Before dispatching subagents, print a timing estimate: Before dispatching any subagents, check which files already have cached extraction results: +SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. + ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -226,7 +228,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -305,7 +307,7 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' " ``` -Save new results to cache: +Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -313,7 +315,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') " ``` @@ -551,15 +554,37 @@ from graphify.detect import save_manifest # Save manifest for --update detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) # In --update mode, 'all_files' carries the full corpus; 'files' is the changed # subset. Full-rebuild mode populates only 'files', so the fallback handles that. # root= relativizes the manifest keys to the scan root (same base as the build), # so the on-disk manifest is portable across clones/machines and a later --update # matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: +# a detected file whose chunk failed or was omitted must stay unstamped so the +# next --update re-queues it, otherwise it is marked done and its content is lost +# forever (#2015). This mirrors the library extract path exactly +# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the +# raw corpus. Code files are always stamped (AST is deterministic); only semantic +# types are gated on output. +from graphify.cli import _stamped_manifest_files +_corpus = detect.get('all_files') or detect['files'] +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +# Files dispatched this run (the changed subset) but NOT stamped above still carry +# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues +# them instead of reading them as unchanged (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root +# files newly excluded since last run are dropped rather than masquerading as +# deletions; untouched files' prior rows are still preserved (#1908). +_scan = {f for fl in _corpus.values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) diff --git a/tools/skillgen/expected/graphify__skill-kiro.md b/tools/skillgen/expected/graphify__skill-kiro.md index b3542438d..d98865cc8 100644 --- a/tools/skillgen/expected/graphify__skill-kiro.md +++ b/tools/skillgen/expected/graphify__skill-kiro.md @@ -10,7 +10,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## Usage ``` -/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) /graphify # full pipeline on specific path /graphify https://github.com// # clone repo then run full pipeline on it /graphify https://github.com// --branch # clone a specific branch @@ -70,7 +70,7 @@ PYTHON="" GRAPHIFY_BIN=$(which graphify 2>/dev/null) # 1. uv tool installs — most reliable on modern Mac/Linux if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi fi # 2. Read shebang from graphify binary (pipx and direct pip installs) @@ -86,7 +86,7 @@ if [ -z "$PYTHON" ]; then PYTHON="python3"; fi if ! "$PYTHON" -c "import graphify" 2>/dev/null; then if command -v uv >/dev/null 2>&1; then uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi else "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ @@ -131,7 +131,7 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). @@ -214,6 +214,8 @@ Before dispatching subagents, print a timing estimate: Before dispatching any subagents, check which files already have cached extraction results: +SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. + ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -226,7 +228,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -305,7 +307,7 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' " ``` -Save new results to cache: +Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -313,7 +315,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') " ``` @@ -551,15 +554,37 @@ from graphify.detect import save_manifest # Save manifest for --update detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) # In --update mode, 'all_files' carries the full corpus; 'files' is the changed # subset. Full-rebuild mode populates only 'files', so the fallback handles that. # root= relativizes the manifest keys to the scan root (same base as the build), # so the on-disk manifest is portable across clones/machines and a later --update # matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: +# a detected file whose chunk failed or was omitted must stay unstamped so the +# next --update re-queues it, otherwise it is marked done and its content is lost +# forever (#2015). This mirrors the library extract path exactly +# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the +# raw corpus. Code files are always stamped (AST is deterministic); only semantic +# types are gated on output. +from graphify.cli import _stamped_manifest_files +_corpus = detect.get('all_files') or detect['files'] +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +# Files dispatched this run (the changed subset) but NOT stamped above still carry +# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues +# them instead of reading them as unchanged (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root +# files newly excluded since last run are dropped rather than masquerading as +# deletions; untouched files' prior rows are still preserved (#1908). +_scan = {f for fl in _corpus.values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) diff --git a/tools/skillgen/expected/graphify__skill-opencode.md b/tools/skillgen/expected/graphify__skill-opencode.md index 684e0e223..cf5dae440 100644 --- a/tools/skillgen/expected/graphify__skill-opencode.md +++ b/tools/skillgen/expected/graphify__skill-opencode.md @@ -10,7 +10,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## Usage ``` -/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) /graphify # full pipeline on specific path /graphify https://github.com// # clone repo then run full pipeline on it /graphify https://github.com// --branch # clone a specific branch @@ -70,7 +70,7 @@ PYTHON="" GRAPHIFY_BIN=$(which graphify 2>/dev/null) # 1. uv tool installs — most reliable on modern Mac/Linux if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi fi # 2. Read shebang from graphify binary (pipx and direct pip installs) @@ -86,7 +86,7 @@ if [ -z "$PYTHON" ]; then PYTHON="python3"; fi if ! "$PYTHON" -c "import graphify" 2>/dev/null; then if command -v uv >/dev/null 2>&1; then uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi else "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ @@ -131,7 +131,7 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). @@ -214,6 +214,8 @@ Before dispatching subagents, print a timing estimate: Before dispatching any subagents, check which files already have cached extraction results: +SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. + ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -226,7 +228,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -297,7 +299,7 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' " ``` -Save new results to cache: +Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -305,7 +307,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') " ``` @@ -543,15 +546,37 @@ from graphify.detect import save_manifest # Save manifest for --update detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) # In --update mode, 'all_files' carries the full corpus; 'files' is the changed # subset. Full-rebuild mode populates only 'files', so the fallback handles that. # root= relativizes the manifest keys to the scan root (same base as the build), # so the on-disk manifest is portable across clones/machines and a later --update # matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: +# a detected file whose chunk failed or was omitted must stay unstamped so the +# next --update re-queues it, otherwise it is marked done and its content is lost +# forever (#2015). This mirrors the library extract path exactly +# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the +# raw corpus. Code files are always stamped (AST is deterministic); only semantic +# types are gated on output. +from graphify.cli import _stamped_manifest_files +_corpus = detect.get('all_files') or detect['files'] +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +# Files dispatched this run (the changed subset) but NOT stamped above still carry +# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues +# them instead of reading them as unchanged (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root +# files newly excluded since last run are dropped rather than masquerading as +# deletions; untouched files' prior rows are still preserved (#1908). +_scan = {f for fl in _corpus.values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) diff --git a/tools/skillgen/expected/graphify__skill-pi.md b/tools/skillgen/expected/graphify__skill-pi.md index b3542438d..d98865cc8 100644 --- a/tools/skillgen/expected/graphify__skill-pi.md +++ b/tools/skillgen/expected/graphify__skill-pi.md @@ -10,7 +10,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## Usage ``` -/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) /graphify # full pipeline on specific path /graphify https://github.com// # clone repo then run full pipeline on it /graphify https://github.com// --branch # clone a specific branch @@ -70,7 +70,7 @@ PYTHON="" GRAPHIFY_BIN=$(which graphify 2>/dev/null) # 1. uv tool installs — most reliable on modern Mac/Linux if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi fi # 2. Read shebang from graphify binary (pipx and direct pip installs) @@ -86,7 +86,7 @@ if [ -z "$PYTHON" ]; then PYTHON="python3"; fi if ! "$PYTHON" -c "import graphify" 2>/dev/null; then if command -v uv >/dev/null 2>&1; then uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi else "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ @@ -131,7 +131,7 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). @@ -214,6 +214,8 @@ Before dispatching subagents, print a timing estimate: Before dispatching any subagents, check which files already have cached extraction results: +SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. + ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -226,7 +228,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -305,7 +307,7 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' " ``` -Save new results to cache: +Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -313,7 +315,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') " ``` @@ -551,15 +554,37 @@ from graphify.detect import save_manifest # Save manifest for --update detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) # In --update mode, 'all_files' carries the full corpus; 'files' is the changed # subset. Full-rebuild mode populates only 'files', so the fallback handles that. # root= relativizes the manifest keys to the scan root (same base as the build), # so the on-disk manifest is portable across clones/machines and a later --update # matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: +# a detected file whose chunk failed or was omitted must stay unstamped so the +# next --update re-queues it, otherwise it is marked done and its content is lost +# forever (#2015). This mirrors the library extract path exactly +# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the +# raw corpus. Code files are always stamped (AST is deterministic); only semantic +# types are gated on output. +from graphify.cli import _stamped_manifest_files +_corpus = detect.get('all_files') or detect['files'] +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +# Files dispatched this run (the changed subset) but NOT stamped above still carry +# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues +# them instead of reading them as unchanged (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root +# files newly excluded since last run are dropped rather than masquerading as +# deletions; untouched files' prior rows are still preserved (#1908). +_scan = {f for fl in _corpus.values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) diff --git a/tools/skillgen/expected/graphify__skill-trae.md b/tools/skillgen/expected/graphify__skill-trae.md index ac16b925d..b0cbeb122 100644 --- a/tools/skillgen/expected/graphify__skill-trae.md +++ b/tools/skillgen/expected/graphify__skill-trae.md @@ -10,7 +10,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## Usage ``` -/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) /graphify # full pipeline on specific path /graphify https://github.com// # clone repo then run full pipeline on it /graphify https://github.com// --branch # clone a specific branch @@ -70,7 +70,7 @@ PYTHON="" GRAPHIFY_BIN=$(which graphify 2>/dev/null) # 1. uv tool installs — most reliable on modern Mac/Linux if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi fi # 2. Read shebang from graphify binary (pipx and direct pip installs) @@ -86,7 +86,7 @@ if [ -z "$PYTHON" ]; then PYTHON="python3"; fi if ! "$PYTHON" -c "import graphify" 2>/dev/null; then if command -v uv >/dev/null 2>&1; then uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi else "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ @@ -131,7 +131,7 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). @@ -214,6 +214,8 @@ Before dispatching subagents, print a timing estimate: Before dispatching any subagents, check which files already have cached extraction results: +SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. + ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -226,7 +228,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -303,7 +305,7 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' " ``` -Save new results to cache: +Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -311,7 +313,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') " ``` @@ -549,15 +552,37 @@ from graphify.detect import save_manifest # Save manifest for --update detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) # In --update mode, 'all_files' carries the full corpus; 'files' is the changed # subset. Full-rebuild mode populates only 'files', so the fallback handles that. # root= relativizes the manifest keys to the scan root (same base as the build), # so the on-disk manifest is portable across clones/machines and a later --update # matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: +# a detected file whose chunk failed or was omitted must stay unstamped so the +# next --update re-queues it, otherwise it is marked done and its content is lost +# forever (#2015). This mirrors the library extract path exactly +# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the +# raw corpus. Code files are always stamped (AST is deterministic); only semantic +# types are gated on output. +from graphify.cli import _stamped_manifest_files +_corpus = detect.get('all_files') or detect['files'] +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +# Files dispatched this run (the changed subset) but NOT stamped above still carry +# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues +# them instead of reading them as unchanged (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root +# files newly excluded since last run are dropped rather than masquerading as +# deletions; untouched files' prior rows are still preserved (#1908). +_scan = {f for fl in _corpus.values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) diff --git a/tools/skillgen/expected/graphify__skill-vscode.md b/tools/skillgen/expected/graphify__skill-vscode.md index 31a3352f7..3e6bc6b7b 100644 --- a/tools/skillgen/expected/graphify__skill-vscode.md +++ b/tools/skillgen/expected/graphify__skill-vscode.md @@ -10,7 +10,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## Usage ``` -/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) /graphify # full pipeline on specific path /graphify https://github.com// # clone repo then run full pipeline on it /graphify https://github.com// --branch # clone a specific branch @@ -70,7 +70,7 @@ PYTHON="" GRAPHIFY_BIN=$(which graphify 2>/dev/null) # 1. uv tool installs — most reliable on modern Mac/Linux if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi fi # 2. Read shebang from graphify binary (pipx and direct pip installs) @@ -86,7 +86,7 @@ if [ -z "$PYTHON" ]; then PYTHON="python3"; fi if ! "$PYTHON" -c "import graphify" 2>/dev/null; then if command -v uv >/dev/null 2>&1; then uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi else "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ @@ -131,7 +131,7 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). @@ -214,6 +214,8 @@ Before dispatching subagents, print a timing estimate: Before dispatching any subagents, check which files already have cached extraction results: +SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. + ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -226,7 +228,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -301,7 +303,7 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' " ``` -Save new results to cache: +Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -309,7 +311,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') " ``` @@ -547,15 +550,37 @@ from graphify.detect import save_manifest # Save manifest for --update detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) # In --update mode, 'all_files' carries the full corpus; 'files' is the changed # subset. Full-rebuild mode populates only 'files', so the fallback handles that. # root= relativizes the manifest keys to the scan root (same base as the build), # so the on-disk manifest is portable across clones/machines and a later --update # matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: +# a detected file whose chunk failed or was omitted must stay unstamped so the +# next --update re-queues it, otherwise it is marked done and its content is lost +# forever (#2015). This mirrors the library extract path exactly +# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the +# raw corpus. Code files are always stamped (AST is deterministic); only semantic +# types are gated on output. +from graphify.cli import _stamped_manifest_files +_corpus = detect.get('all_files') or detect['files'] +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +# Files dispatched this run (the changed subset) but NOT stamped above still carry +# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues +# them instead of reading them as unchanged (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root +# files newly excluded since last run are dropped rather than masquerading as +# deletions; untouched files' prior rows are still preserved (#1908). +_scan = {f for fl in _corpus.values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) diff --git a/tools/skillgen/expected/graphify__skill-windows.md b/tools/skillgen/expected/graphify__skill-windows.md index 5a55c577b..574384576 100644 --- a/tools/skillgen/expected/graphify__skill-windows.md +++ b/tools/skillgen/expected/graphify__skill-windows.md @@ -10,7 +10,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## Usage ``` -/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) /graphify # full pipeline on specific path /graphify https://github.com// # clone repo then run full pipeline on it /graphify https://github.com// --branch # clone a specific branch @@ -153,7 +153,7 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). @@ -236,6 +236,8 @@ Before dispatching subagents, print a timing estimate: Before dispatching any subagents, check which files already have cached extraction results: +SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. + ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -248,7 +250,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -327,7 +329,7 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' " ``` -Save new results to cache: +Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -335,7 +337,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') " ``` @@ -573,15 +576,37 @@ from graphify.detect import save_manifest # Save manifest for --update detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) # In --update mode, 'all_files' carries the full corpus; 'files' is the changed # subset. Full-rebuild mode populates only 'files', so the fallback handles that. # root= relativizes the manifest keys to the scan root (same base as the build), # so the on-disk manifest is portable across clones/machines and a later --update # matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: +# a detected file whose chunk failed or was omitted must stay unstamped so the +# next --update re-queues it, otherwise it is marked done and its content is lost +# forever (#2015). This mirrors the library extract path exactly +# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the +# raw corpus. Code files are always stamped (AST is deterministic); only semantic +# types are gated on output. +from graphify.cli import _stamped_manifest_files +_corpus = detect.get('all_files') or detect['files'] +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +# Files dispatched this run (the changed subset) but NOT stamped above still carry +# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues +# them instead of reading them as unchanged (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root +# files newly excluded since last run are dropped rather than masquerading as +# deletions; untouched files' prior rows are still preserved (#1908). +_scan = {f for fl in _corpus.values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) diff --git a/tools/skillgen/expected/graphify__skill.md b/tools/skillgen/expected/graphify__skill.md index b3542438d..d98865cc8 100644 --- a/tools/skillgen/expected/graphify__skill.md +++ b/tools/skillgen/expected/graphify__skill.md @@ -10,7 +10,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## Usage ``` -/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) /graphify # full pipeline on specific path /graphify https://github.com// # clone repo then run full pipeline on it /graphify https://github.com// --branch # clone a specific branch @@ -70,7 +70,7 @@ PYTHON="" GRAPHIFY_BIN=$(which graphify 2>/dev/null) # 1. uv tool installs — most reliable on modern Mac/Linux if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi fi # 2. Read shebang from graphify binary (pipx and direct pip installs) @@ -86,7 +86,7 @@ if [ -z "$PYTHON" ]; then PYTHON="python3"; fi if ! "$PYTHON" -c "import graphify" 2>/dev/null; then if command -v uv >/dev/null 2>&1; then uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi else "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ @@ -131,7 +131,7 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). @@ -214,6 +214,8 @@ Before dispatching subagents, print a timing estimate: Before dispatching any subagents, check which files already have cached extraction results: +SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. + ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -226,7 +228,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -305,7 +307,7 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' " ``` -Save new results to cache: +Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -313,7 +315,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') " ``` @@ -551,15 +554,37 @@ from graphify.detect import save_manifest # Save manifest for --update detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) # In --update mode, 'all_files' carries the full corpus; 'files' is the changed # subset. Full-rebuild mode populates only 'files', so the fallback handles that. # root= relativizes the manifest keys to the scan root (same base as the build), # so the on-disk manifest is portable across clones/machines and a later --update # matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: +# a detected file whose chunk failed or was omitted must stay unstamped so the +# next --update re-queues it, otherwise it is marked done and its content is lost +# forever (#2015). This mirrors the library extract path exactly +# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the +# raw corpus. Code files are always stamped (AST is deterministic); only semantic +# types are gated on output. +from graphify.cli import _stamped_manifest_files +_corpus = detect.get('all_files') or detect['files'] +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +# Files dispatched this run (the changed subset) but NOT stamped above still carry +# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues +# them instead of reading them as unchanged (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root +# files newly excluded since last run are dropped rather than masquerading as +# deletions; untouched files' prior rows are still preserved (#1908). +_scan = {f for fl in _corpus.values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) diff --git a/tools/skillgen/expected/graphify__skills__agents__references__update.md b/tools/skillgen/expected/graphify__skills__agents__references__update.md index fa2612180..3632fd412 100644 --- a/tools/skillgen/expected/graphify__skills__agents__references__update.md +++ b/tools/skillgen/expected/graphify__skills__agents__references__update.md @@ -142,7 +142,25 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (new_extraction is this run's fresh extraction, read above before the +# merge overwrote the file): a changed doc whose chunk failed must stay unstamped +# so the next --update re-queues it, otherwise it is marked done and its content +# is lost forever (#2015). Mirrors the library extract path +# (cli._stamped_manifest_files + clear_semantic + scan_corpus). +from graphify.cli import _stamped_manifest_files +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " ``` diff --git a/tools/skillgen/expected/graphify__skills__amp__references__update.md b/tools/skillgen/expected/graphify__skills__amp__references__update.md index fa2612180..3632fd412 100644 --- a/tools/skillgen/expected/graphify__skills__amp__references__update.md +++ b/tools/skillgen/expected/graphify__skills__amp__references__update.md @@ -142,7 +142,25 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (new_extraction is this run's fresh extraction, read above before the +# merge overwrote the file): a changed doc whose chunk failed must stay unstamped +# so the next --update re-queues it, otherwise it is marked done and its content +# is lost forever (#2015). Mirrors the library extract path +# (cli._stamped_manifest_files + clear_semantic + scan_corpus). +from graphify.cli import _stamped_manifest_files +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " ``` diff --git a/tools/skillgen/expected/graphify__skills__claude__references__update.md b/tools/skillgen/expected/graphify__skills__claude__references__update.md index fa2612180..3632fd412 100644 --- a/tools/skillgen/expected/graphify__skills__claude__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claude__references__update.md @@ -142,7 +142,25 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (new_extraction is this run's fresh extraction, read above before the +# merge overwrote the file): a changed doc whose chunk failed must stay unstamped +# so the next --update re-queues it, otherwise it is marked done and its content +# is lost forever (#2015). Mirrors the library extract path +# (cli._stamped_manifest_files + clear_semantic + scan_corpus). +from graphify.cli import _stamped_manifest_files +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " ``` diff --git a/tools/skillgen/expected/graphify__skills__claw__references__update.md b/tools/skillgen/expected/graphify__skills__claw__references__update.md index fa2612180..3632fd412 100644 --- a/tools/skillgen/expected/graphify__skills__claw__references__update.md +++ b/tools/skillgen/expected/graphify__skills__claw__references__update.md @@ -142,7 +142,25 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (new_extraction is this run's fresh extraction, read above before the +# merge overwrote the file): a changed doc whose chunk failed must stay unstamped +# so the next --update re-queues it, otherwise it is marked done and its content +# is lost forever (#2015). Mirrors the library extract path +# (cli._stamped_manifest_files + clear_semantic + scan_corpus). +from graphify.cli import _stamped_manifest_files +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " ``` diff --git a/tools/skillgen/expected/graphify__skills__codex__references__update.md b/tools/skillgen/expected/graphify__skills__codex__references__update.md index fa2612180..3632fd412 100644 --- a/tools/skillgen/expected/graphify__skills__codex__references__update.md +++ b/tools/skillgen/expected/graphify__skills__codex__references__update.md @@ -142,7 +142,25 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (new_extraction is this run's fresh extraction, read above before the +# merge overwrote the file): a changed doc whose chunk failed must stay unstamped +# so the next --update re-queues it, otherwise it is marked done and its content +# is lost forever (#2015). Mirrors the library extract path +# (cli._stamped_manifest_files + clear_semantic + scan_corpus). +from graphify.cli import _stamped_manifest_files +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " ``` diff --git a/tools/skillgen/expected/graphify__skills__copilot__references__update.md b/tools/skillgen/expected/graphify__skills__copilot__references__update.md index fa2612180..3632fd412 100644 --- a/tools/skillgen/expected/graphify__skills__copilot__references__update.md +++ b/tools/skillgen/expected/graphify__skills__copilot__references__update.md @@ -142,7 +142,25 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (new_extraction is this run's fresh extraction, read above before the +# merge overwrote the file): a changed doc whose chunk failed must stay unstamped +# so the next --update re-queues it, otherwise it is marked done and its content +# is lost forever (#2015). Mirrors the library extract path +# (cli._stamped_manifest_files + clear_semantic + scan_corpus). +from graphify.cli import _stamped_manifest_files +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " ``` diff --git a/tools/skillgen/expected/graphify__skills__droid__references__update.md b/tools/skillgen/expected/graphify__skills__droid__references__update.md index fa2612180..3632fd412 100644 --- a/tools/skillgen/expected/graphify__skills__droid__references__update.md +++ b/tools/skillgen/expected/graphify__skills__droid__references__update.md @@ -142,7 +142,25 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (new_extraction is this run's fresh extraction, read above before the +# merge overwrote the file): a changed doc whose chunk failed must stay unstamped +# so the next --update re-queues it, otherwise it is marked done and its content +# is lost forever (#2015). Mirrors the library extract path +# (cli._stamped_manifest_files + clear_semantic + scan_corpus). +from graphify.cli import _stamped_manifest_files +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " ``` diff --git a/tools/skillgen/expected/graphify__skills__kilo__references__update.md b/tools/skillgen/expected/graphify__skills__kilo__references__update.md index fa2612180..3632fd412 100644 --- a/tools/skillgen/expected/graphify__skills__kilo__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kilo__references__update.md @@ -142,7 +142,25 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (new_extraction is this run's fresh extraction, read above before the +# merge overwrote the file): a changed doc whose chunk failed must stay unstamped +# so the next --update re-queues it, otherwise it is marked done and its content +# is lost forever (#2015). Mirrors the library extract path +# (cli._stamped_manifest_files + clear_semantic + scan_corpus). +from graphify.cli import _stamped_manifest_files +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " ``` diff --git a/tools/skillgen/expected/graphify__skills__kiro__references__update.md b/tools/skillgen/expected/graphify__skills__kiro__references__update.md index fa2612180..3632fd412 100644 --- a/tools/skillgen/expected/graphify__skills__kiro__references__update.md +++ b/tools/skillgen/expected/graphify__skills__kiro__references__update.md @@ -142,7 +142,25 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (new_extraction is this run's fresh extraction, read above before the +# merge overwrote the file): a changed doc whose chunk failed must stay unstamped +# so the next --update re-queues it, otherwise it is marked done and its content +# is lost forever (#2015). Mirrors the library extract path +# (cli._stamped_manifest_files + clear_semantic + scan_corpus). +from graphify.cli import _stamped_manifest_files +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " ``` diff --git a/tools/skillgen/expected/graphify__skills__opencode__references__update.md b/tools/skillgen/expected/graphify__skills__opencode__references__update.md index fa2612180..3632fd412 100644 --- a/tools/skillgen/expected/graphify__skills__opencode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__opencode__references__update.md @@ -142,7 +142,25 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (new_extraction is this run's fresh extraction, read above before the +# merge overwrote the file): a changed doc whose chunk failed must stay unstamped +# so the next --update re-queues it, otherwise it is marked done and its content +# is lost forever (#2015). Mirrors the library extract path +# (cli._stamped_manifest_files + clear_semantic + scan_corpus). +from graphify.cli import _stamped_manifest_files +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " ``` diff --git a/tools/skillgen/expected/graphify__skills__pi__references__update.md b/tools/skillgen/expected/graphify__skills__pi__references__update.md index fa2612180..3632fd412 100644 --- a/tools/skillgen/expected/graphify__skills__pi__references__update.md +++ b/tools/skillgen/expected/graphify__skills__pi__references__update.md @@ -142,7 +142,25 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (new_extraction is this run's fresh extraction, read above before the +# merge overwrote the file): a changed doc whose chunk failed must stay unstamped +# so the next --update re-queues it, otherwise it is marked done and its content +# is lost forever (#2015). Mirrors the library extract path +# (cli._stamped_manifest_files + clear_semantic + scan_corpus). +from graphify.cli import _stamped_manifest_files +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " ``` diff --git a/tools/skillgen/expected/graphify__skills__trae__references__update.md b/tools/skillgen/expected/graphify__skills__trae__references__update.md index fa2612180..3632fd412 100644 --- a/tools/skillgen/expected/graphify__skills__trae__references__update.md +++ b/tools/skillgen/expected/graphify__skills__trae__references__update.md @@ -142,7 +142,25 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (new_extraction is this run's fresh extraction, read above before the +# merge overwrote the file): a changed doc whose chunk failed must stay unstamped +# so the next --update re-queues it, otherwise it is marked done and its content +# is lost forever (#2015). Mirrors the library extract path +# (cli._stamped_manifest_files + clear_semantic + scan_corpus). +from graphify.cli import _stamped_manifest_files +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " ``` diff --git a/tools/skillgen/expected/graphify__skills__vscode__references__update.md b/tools/skillgen/expected/graphify__skills__vscode__references__update.md index fa2612180..3632fd412 100644 --- a/tools/skillgen/expected/graphify__skills__vscode__references__update.md +++ b/tools/skillgen/expected/graphify__skills__vscode__references__update.md @@ -142,7 +142,25 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (new_extraction is this run's fresh extraction, read above before the +# merge overwrote the file): a changed doc whose chunk failed must stay unstamped +# so the next --update re-queues it, otherwise it is marked done and its content +# is lost forever (#2015). Mirrors the library extract path +# (cli._stamped_manifest_files + clear_semantic + scan_corpus). +from graphify.cli import _stamped_manifest_files +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " ``` diff --git a/tools/skillgen/expected/graphify__skills__windows__references__update.md b/tools/skillgen/expected/graphify__skills__windows__references__update.md index fa2612180..3632fd412 100644 --- a/tools/skillgen/expected/graphify__skills__windows__references__update.md +++ b/tools/skillgen/expected/graphify__skills__windows__references__update.md @@ -142,7 +142,25 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (new_extraction is this run's fresh extraction, read above before the +# merge overwrote the file): a changed doc whose chunk failed must stay unstamped +# so the next --update re-queues it, otherwise it is marked done and its content +# is lost forever (#2015). Mirrors the library extract path +# (cli._stamped_manifest_files + clear_semantic + scan_corpus). +from graphify.cli import _stamped_manifest_files +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " ``` diff --git a/tools/skillgen/fragments/core/aider.md b/tools/skillgen/fragments/core/aider.md index 2ccf5326c..4f03ccbae 100644 --- a/tools/skillgen/fragments/core/aider.md +++ b/tools/skillgen/fragments/core/aider.md @@ -10,7 +10,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## Usage ``` -/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) /graphify # full pipeline on specific path /graphify --mode deep # thorough extraction, richer INFERRED edges /graphify --update # incremental - re-extract only new/changed files @@ -64,7 +64,7 @@ PYTHON="" GRAPHIFY_BIN=$(which graphify 2>/dev/null) # 1. uv tool installs — most reliable on modern Mac/Linux if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi fi # 2. Read shebang from graphify binary (pipx and direct pip installs) @@ -80,7 +80,7 @@ if [ -z "$PYTHON" ]; then PYTHON="python3"; fi if ! "$PYTHON" -c "import graphify" 2>/dev/null; then if command -v uv >/dev/null 2>&1; then uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi else "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ @@ -123,7 +123,7 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. - Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. @@ -313,7 +313,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('.graphify_semantic_new.json').read_text()) if Path('.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +uncached = [line for line in Path('.graphify_uncached.txt').read_text().splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached) print(f'Cached {saved} files') " ``` @@ -676,10 +677,19 @@ from graphify.detect import save_manifest # Save manifest for --update detect = json.loads(Path('.graphify_detect.json').read_text()) -save_manifest(detect['files'], root='INPUT_PATH') +extract = json.loads(Path('.graphify_extract.json').read_text()) +# Stamp only semantic files that produced output so a failed chunk is re-queued next run, not lost (#2015). +from graphify.cli import _stamped_manifest_files +_corpus = detect.get('all_files') or detect['files'] +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +_scan = {f for fl in _corpus.values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker -extract = json.loads(Path('.graphify_extract.json').read_text()) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) diff --git a/tools/skillgen/fragments/core/core.md b/tools/skillgen/fragments/core/core.md index 0516b3a6e..e28910728 100644 --- a/tools/skillgen/fragments/core/core.md +++ b/tools/skillgen/fragments/core/core.md @@ -7,7 +7,7 @@ Turn any folder of files into a navigable knowledge graph with community detecti ## Usage ``` -/graphify # full pipeline on current directory → Obsidian vault +/graphify # full pipeline on current directory (HTML viz; add --obsidian for a vault) /graphify # full pipeline on specific path /graphify https://github.com// # clone repo then run full pipeline on it /graphify https://github.com// --branch # clone a specific branch @@ -90,7 +90,7 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 500: show the warning. Then compute the top 5 first-level subdirectories by file count: - Read `scan_root` from the detect JSON (always an absolute path to the resolved INPUT_PATH). - Concatenate all file lists across all types (`code`, `document`, `paper`, `image`, `video`). @@ -173,6 +173,8 @@ Before dispatching subagents, print a timing estimate: Before dispatching any subagents, check which files already have cached extraction results: +SPEC_PATH below is the **absolute** path of the `references/extraction-spec.md` that ships beside this SKILL.md — the same file Step B2 loads and hands to every subagent. It is the extraction prompt, so cache entries are attributed to it: when a graphify upgrade changes the prompt, entries produced by the old one are re-extracted instead of replayed, and unchanged prompts keep their entries (#1939). Substitute the real path in both Step B0 and Step B3 — pass the same one to each, and do not drop the argument. + ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -185,7 +187,7 @@ detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encodin # every source file (#1392). Video is transcribed to a document in Step 2.5 first. all_files = [f for cat in ('document', 'paper', 'image') for f in detect['files'].get(cat, [])] -cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH') +cached_nodes, cached_edges, cached_hyperedges, uncached = check_semantic_cache(all_files, root='INPUT_PATH', prompt_file='SPEC_PATH') # Always (re)write the cache file: write hits, else DELETE any leftover from a prior # run so Part C never merges a stale .graphify_cached.json (#1392). @@ -240,7 +242,7 @@ print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens' " ``` -Save new results to cache: +Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939): ```bash $(cat graphify-out/.graphify_python) -c " import json @@ -248,7 +250,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH') +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root='INPUT_PATH', allowed_source_files=uncached, prompt_file='SPEC_PATH') print(f'Cached {saved} files') " ``` @@ -486,15 +489,37 @@ from graphify.detect import save_manifest # Save manifest for --update detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\")) +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) # In --update mode, 'all_files' carries the full corpus; 'files' is the changed # subset. Full-rebuild mode populates only 'files', so the fallback handles that. # root= relativizes the manifest keys to the scan root (same base as the build), # so the on-disk manifest is portable across clones/machines and a later --update # matches cached files instead of missing every one (#1417). -save_manifest(detect.get('all_files') or detect['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output: +# a detected file whose chunk failed or was omitted must stay unstamped so the +# next --update re-queues it, otherwise it is marked done and its content is lost +# forever (#2015). This mirrors the library extract path exactly +# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the +# raw corpus. Code files are always stamped (AST is deterministic); only semantic +# types are gated on output. +from graphify.cli import _stamped_manifest_files +_corpus = detect.get('all_files') or detect['files'] +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +# Files dispatched this run (the changed subset) but NOT stamped above still carry +# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues +# them instead of reading them as unchanged (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root +# files newly excluded since last run are dropped rather than masquerading as +# deletions; untouched files' prior rows are still preserved (#1908). +_scan = {f for fl in _corpus.values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\")) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) diff --git a/tools/skillgen/fragments/core/devin.md b/tools/skillgen/fragments/core/devin.md index 13fb37c85..e3e6d2dec 100644 --- a/tools/skillgen/fragments/core/devin.md +++ b/tools/skillgen/fragments/core/devin.md @@ -75,7 +75,7 @@ PYTHON="" GRAPHIFY_BIN=$(which graphify 2>/dev/null) # 1. uv tool installs — most reliable on modern Mac/Linux if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi fi # 2. Read shebang from graphify binary (pipx and direct pip installs) @@ -91,7 +91,7 @@ if [ -z "$PYTHON" ]; then PYTHON="python3"; fi if ! "$PYTHON" -c "import graphify" 2>/dev/null; then if command -v uv >/dev/null 2>&1; then uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi else "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ @@ -136,7 +136,7 @@ Omit any category with 0 files from the summary. Then act on it: - If `total_files` is 0: stop with "No supported files found in [path]." -- If `skipped_sensitive` is non-empty: mention file count skipped, not the file names. +- If `skipped_sensitive` is non-empty: report the count and list the skipped file names, so a wrongly-flagged source or doc is visible and can be renamed or moved (#2106). - If `total_words` > 2,000,000 OR `total_files` > 200: show the warning and the top 5 subdirectories by file count, then ask which subfolder to run on. Wait for the user's answer before proceeding. - Otherwise: proceed directly to Step 2.5 if video files were detected, or Step 3 if not. @@ -374,7 +374,8 @@ from graphify.cache import save_semantic_cache from pathlib import Path new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text()) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]} -saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', [])) +uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text().splitlines() if line] +saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), allowed_source_files=uncached) print(f'Cached {saved} files') " ``` @@ -794,10 +795,19 @@ from graphify.detect import save_manifest # Save manifest for --update detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text()) -save_manifest(detect['files'], root='INPUT_PATH') +extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) +# Stamp only semantic files that produced output so a failed chunk is re-queued next run, not lost (#2015). +from graphify.cli import _stamped_manifest_files +_corpus = detect.get('all_files') or detect['files'] +_manifest_files = _stamped_manifest_files(_corpus, extract, Path('INPUT_PATH')) +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +_scan = {f for fl in _corpus.values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) # Update cumulative cost tracker -extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text()) input_tok = extract.get('input_tokens', 0) output_tok = extract.get('output_tokens', 0) diff --git a/tools/skillgen/fragments/references/shared/update.md b/tools/skillgen/fragments/references/shared/update.md index fa2612180..3632fd412 100644 --- a/tools/skillgen/fragments/references/shared/update.md +++ b/tools/skillgen/fragments/references/shared/update.md @@ -142,7 +142,25 @@ print(f'[graphify update] Merged extraction written ({len(merged_out[\"nodes\"]) # root= matches the build_merge call above so the manifest keys stay relative to # the scan root — portable across clones/machines, so --update keeps matching # cached files instead of missing every one after a move (#1417). -save_manifest(incremental['files'], root='INPUT_PATH') +# +# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output +# THIS run (new_extraction is this run's fresh extraction, read above before the +# merge overwrote the file): a changed doc whose chunk failed must stay unstamped +# so the next --update re-queues it, otherwise it is marked done and its content +# is lost forever (#2015). Mirrors the library extract path +# (cli._stamped_manifest_files + clear_semantic + scan_corpus). +from graphify.cli import _stamped_manifest_files +_manifest_files = _stamped_manifest_files(incremental['files'], new_extraction, Path('INPUT_PATH')) +# Changed semantic files dispatched this run but NOT stamped had their chunk fail +# or be omitted; clear any stale semantic_hash so they are re-queued (#1948). +_sem_types = ('document', 'paper', 'image') +_dispatched = {f for t, fl in incremental.get('new_files', {}).items() if t in _sem_types for f in fl} +_stamped = {f for fl in _manifest_files.values() for f in fl} +_cleared = _dispatched - _stamped +# scan_corpus = the RAW full corpus so in-root files newly excluded since last run +# are dropped rather than masquerading as deletions; untouched rows preserved (#1908). +_scan = {f for fl in incremental['files'].values() for f in fl} +save_manifest(_manifest_files, root='INPUT_PATH', scan_corpus=_scan, clear_semantic=_cleared or None) print('[graphify update] Manifest saved.') " ``` diff --git a/tools/skillgen/fragments/shell/posix.md b/tools/skillgen/fragments/shell/posix.md index 717bd800a..3534417d2 100644 --- a/tools/skillgen/fragments/shell/posix.md +++ b/tools/skillgen/fragments/shell/posix.md @@ -4,7 +4,7 @@ PYTHON="" GRAPHIFY_BIN=$(which graphify 2>/dev/null) # 1. uv tool installs — most reliable on modern Mac/Linux if [ -z "$PYTHON" ] && command -v uv >/dev/null 2>&1; then - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi fi # 2. Read shebang from graphify binary (pipx and direct pip installs) @@ -20,7 +20,7 @@ if [ -z "$PYTHON" ]; then PYTHON="python3"; fi if ! "$PYTHON" -c "import graphify" 2>/dev/null; then if command -v uv >/dev/null 2>&1; then uv tool install --upgrade graphifyy -q 2>&1 | tail -3 - _UV_PY=$(uv tool run graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) + _UV_PY=$(uv tool run --from graphifyy python -c "import sys; print(sys.executable)" 2>/dev/null) if [ -n "$_UV_PY" ]; then PYTHON="$_UV_PY"; fi else "$PYTHON" -m pip install graphifyy -q 2>/dev/null \ diff --git a/tools/skillgen/gen.py b/tools/skillgen/gen.py index 0082e9e51..732215e8a 100644 --- a/tools/skillgen/gen.py +++ b/tools/skillgen/gen.py @@ -851,6 +851,42 @@ def _is_manifest_root_fix_line(line: str) -> bool: return "save_manifest(" in line and "import" not in line +def _is_manifest_stamp_fix_line(line: str) -> bool: + """Whether a line is part of the manifest over-stamping fix (#2015). + + Step 9 stamped the whole detected corpus, so a semantic file whose chunk + failed (or was omitted) was marked done and never re-queued on the next + ``--update`` — its content lost forever. The manifest is now built with + ``cli._stamped_manifest_files`` (only files that actually produced output) + plus ``clear_semantic``/``scan_corpus``, mirroring the native + ``graphify extract`` path. The rooted ``save_manifest`` call itself is + covered by ``_is_manifest_root_fix_line``; these are the added helper import + and derivation lines, plus the single ``#2015`` explanatory comment. + """ + stripped = line.strip() + return ( + "_stamped_manifest_files" in stripped + or stripped.startswith(( + "_corpus =", + "_manifest_files =", + "_sem_types =", + "_dispatched =", + "_stamped =", + "_cleared =", + "_scan =", + )) + or (stripped.startswith("#") and "#2015" in stripped) + ) + + +def _is_sensitive_reporting_fix_line(line: str) -> bool: + """The #2106 change to how a non-empty ``skipped_sensitive`` is reported: the + skill now lists the skipped file names instead of only a count, so a + wrongly-flagged source/doc is visible. Both the removed count-only line and + the added list-the-names line mention ``skipped_sensitive`` is non-empty.""" + return "skipped_sensitive` is non-empty" in line + + def _is_no_api_key_fix_line(line: str) -> bool: """Whether a line is part of the "no API key required" clarity (#1461). @@ -877,6 +913,47 @@ def _is_shebang_allowlist_fix_line(line: str) -> bool: return "[!a-zA-Z0-9/_." in line +def _is_obsidian_usage_comment_line(line: str) -> bool: + """Whether a line is part of the ``/graphify`` usage-comment fix (#1681). + + The Usage block's bare ``/graphify`` comment said "full pipeline on current + directory -> Obsidian vault", contradicting Step 6 (HTML always; Obsidian vault + only when ``--obsidian`` is explicitly given). The comment now describes the + real default. Both the old (removed) and new (added) comment forms match here. + """ + return "# full pipeline on current directory" in line + + +def _is_uv_from_interpreter_fix_line(line: str) -> bool: + """Whether a line is part of the uv interpreter-detection fix (#1735). + + Step 1's POSIX interpreter probe ran ``uv tool run graphifyy python -c ...``, + but ``graphifyy`` exposes its executable as ``graphify``, so uv treated + ``python`` as a missing ``graphifyy`` command and the probe silently failed + (the ``2>/dev/null`` swallowed uv's "use --from" hint), leaving PYTHON on a + graphify-less system interpreter. The probe now runs + ``uv tool run --from graphifyy python -c ...``. Both the old (removed) and new + (added) forms match here. + """ + return "uv tool run" in line and "graphifyy python" in line + + +def _is_semantic_cache_scope_fix_line(line: str) -> bool: + """Whether a line scopes semantic cache writes to dispatched files (#1757). + + A semantic subagent can mention a corpus file outside its assigned chunk and + misattribute a node to that file. The final cache write now passes the B0 + uncached-file list as an allowlist, so an incidental mention cannot replace + another file's complete cached extraction. Both the old unscoped call + (removed) and the allowlist read/call (added) are sanctioned here. + """ + stripped = line.strip() + return ( + stripped.startswith("uncached = [line for line in Path(") + and ".graphify_uncached.txt" in stripped + ) or stripped.startswith("saved = save_semantic_cache(") + + # Every line that may differ between a rendered monolith and its pristine v8 # baseline. Each predicate documents one sanctioned change-class; a blank line is # allowed because the multi-line fix blocks insert spacing. Anything else failing @@ -890,8 +967,13 @@ def _is_shebang_allowlist_fix_line(line: str) -> bool: _is_cache_unlink_fix_line, _is_zero_node_guard_fix_line, _is_manifest_root_fix_line, + _is_manifest_stamp_fix_line, + _is_sensitive_reporting_fix_line, _is_no_api_key_fix_line, _is_shebang_allowlist_fix_line, + _is_obsidian_usage_comment_line, + _is_uv_from_interpreter_fix_line, + _is_semantic_cache_scope_fix_line, ) @@ -908,8 +990,9 @@ def monolith_roundtrip(platform: Platform) -> list[str]: arbitrary edit (even a blessed one) from drifting them. Sanctioned changes are enumerated as predicates in ``_SANCTIONED_MONOLITH_DIFFS``: the file_type enum unification, the unified frontmatter description, the chunk-cleanup rewrite - (#1172), and the four #1392 runbook fixes (directed propagation, content-only - semantic scope, stale-cache unlink, and the zero-node/shrink-guard ordering). + (#1172), the four #1392 runbook fixes (directed propagation, content-only + semantic scope, stale-cache unlink, and the zero-node/shrink-guard ordering), + and semantic-cache source scoping (#1757). The comparison is a multiset diff, not a positional zip: a line whose text is unchanged but merely *moved* (the report-write line shifted below ``to_json`` diff --git a/uv.lock b/uv.lock index 088ebbbdc..671c304f9 100644 --- a/uv.lock +++ b/uv.lock @@ -1090,7 +1090,7 @@ wheels = [ [[package]] name = "graphifyy" -version = "0.9.6" +version = "0.9.28" source = { editable = "." } dependencies = [ { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },