diff --git a/libs/openant-core/context/application_context.py b/libs/openant-core/context/application_context.py index 93b45d33..f459e51a 100644 --- a/libs/openant-core/context/application_context.py +++ b/libs/openant-core/context/application_context.py @@ -147,6 +147,26 @@ def get_type_info(self) -> dict: """Get detailed information about this application type.""" return APPLICATION_TYPE_INFO.get(self.application_type, {}) + def suppress_local_only(self) -> bool: + """Whether to tell the analyzer to flag only REMOTE-attacker vulnerabilities. + + The "local users have access, only flag remote" framing is correct for a + CLI/library whose inputs are all operator-controlled. But a data-processing + library (parser, deserializer, codec) takes UNTRUSTED INPUT DATA — and that + data crossing into the code IS the attack surface, even with no network + listener. ``requires_remote_trigger`` alone (False for every library) would + suppress exactly those bugs. Gate on the already-captured ``trust_boundaries``: + if any input source is ``untrusted``, do NOT suppress, regardless of type. + """ + if self.requires_remote_trigger: + return False + # Case-insensitive: trust_boundaries values are LLM-generated and may + # deviate from the schema's lowercase 'untrusted' (e.g. 'Untrusted'). + return not any( + str(level).lower() == "untrusted" + for level in (self.trust_boundaries or {}).values() + ) + # Files to check for manual override (in order of priority) MANUAL_OVERRIDE_FILES = [ @@ -488,7 +508,7 @@ def _build_type_descriptions() -> str: **Guidelines:** - `application_type`: MUST be one of: web_app, cli_tool, library, agent_framework, unsupported -- `requires_remote_trigger`: Set to `false` for cli_tool, library, agent_framework. Set to `true` for web_app. +- `requires_remote_trigger`: Set to `true` for web_app, AND for any cli_tool/library/agent_framework that PROCESSES UNTRUSTED INPUT DATA (a parser, deserializer, codec, file/format reader, or anything where `trust_boundaries` marks an input source `untrusted` — the untrusted data crossing into the code is the attack surface even with no network listener). Set to `false` only when every input source is operator-controlled/trusted. - `confidence`: 0.0-1.0 based on how much information was available. - Be specific in `not_a_vulnerability` - these will directly prevent false positives. """ @@ -644,7 +664,7 @@ def format_context_for_prompt(context: ApplicationContext) -> str: lines.append(f"- {item}") lines.append("") - if not context.requires_remote_trigger: + if context.suppress_local_only(): lines.append("**IMPORTANT:** This is a CLI tool/library. Users running this code have local access.") lines.append("Only flag vulnerabilities that could be exploited by a REMOTE attacker, not by local users.") lines.append("") diff --git a/libs/openant-core/core/parser_adapter.py b/libs/openant-core/core/parser_adapter.py index ef2f845a..cf221add 100644 --- a/libs/openant-core/core/parser_adapter.py +++ b/libs/openant-core/core/parser_adapter.py @@ -80,6 +80,7 @@ def parse_repository( name: str = None, diff_manifest: str | None = None, fresh: bool = False, + library_mode: bool = False, ) -> ParseResult: """Parse a repository into an OpenAnt dataset. @@ -96,6 +97,8 @@ def parse_repository( fresh: If True, delete existing dataset.json before parsing so all units are regenerated from scratch. Only dataset.json is deleted; other artifacts in output_dir (e.g. analyzer outputs) are preserved. + library_mode: If True, seed the public API surface as reachability + entry points (opt-in, union-only). Returns: ParseResult with paths to generated files and stats. @@ -127,19 +130,19 @@ def parse_repository( # Dispatch to the right parser if language == "python": - result = _parse_python(repo_path, output_dir, processing_level, skip_tests, name) + result = _parse_python(repo_path, output_dir, processing_level, skip_tests, name, library_mode) elif language == "javascript": - result = _parse_javascript(repo_path, output_dir, processing_level, skip_tests, name) + result = _parse_javascript(repo_path, output_dir, processing_level, skip_tests, name, library_mode) elif language == "go": - result = _parse_go(repo_path, output_dir, processing_level, skip_tests, name) + result = _parse_go(repo_path, output_dir, processing_level, skip_tests, name, library_mode) elif language == "c": - result = _parse_c(repo_path, output_dir, processing_level, skip_tests, name) + result = _parse_c(repo_path, output_dir, processing_level, skip_tests, name, library_mode) elif language == "ruby": - result = _parse_ruby(repo_path, output_dir, processing_level, skip_tests, name) + result = _parse_ruby(repo_path, output_dir, processing_level, skip_tests, name, library_mode) elif language == "php": - result = _parse_php(repo_path, output_dir, processing_level, skip_tests, name) + result = _parse_php(repo_path, output_dir, processing_level, skip_tests, name, library_mode) elif language == "zig": - result = _parse_zig(repo_path, output_dir, processing_level, skip_tests, name) + result = _parse_zig(repo_path, output_dir, processing_level, skip_tests, name, library_mode) else: raise ValueError(f"Unsupported language: {language}") @@ -207,11 +210,18 @@ def _maybe_apply_diff_filter( # Reachability filter (shared by Python path; JS/Go handle it internally) # --------------------------------------------------------------------------- +# library_seed_ids is now shared in utilities/agentic_enhancer/entry_point_detector.py +# so every parser pipeline (not just Python) can seed the public API. It is loaded +# below via the same importlib path as EntryPointDetector to dodge the heavy +# utilities/__init__ imports. + + def apply_reachability_filter( dataset: dict, output_dir: str, processing_level: str, extra_entry_points: "set[str] | None" = None, + library_mode: bool = False, ) -> dict: """Filter dataset units to only those reachable from entry points. @@ -254,6 +264,8 @@ def _load_module(name, filename): _epd = _load_module("entry_point_detector", "entry_point_detector.py") _ra = _load_module("reachability_analyzer", "reachability_analyzer.py") EntryPointDetector = _epd.EntryPointDetector + blackout_warning = _epd.blackout_warning + library_seed_ids = _epd.library_seed_ids ReachabilityAnalyzer = _ra.ReachabilityAnalyzer call_graph_path = os.path.join(output_dir, "call_graph.json") @@ -277,6 +289,11 @@ def _load_module(name, filename): entry_points = detector.detect_entry_points() if extra_entry_points: entry_points = entry_points | extra_entry_points + # Library-mode (opt-in): the public API is the entry surface. Union-only — + # never demotes a structurally-detected app entry point, so an app scan with + # the flag on can only gain reachable units, never lose one. + if library_mode: + entry_points = entry_points | library_seed_ids(functions) units = dataset.get("units", []) original_count = len(units) @@ -349,6 +366,12 @@ def _load_module(name, filename): file=sys.stderr, ) + _blackout = blackout_warning(detector.entry_point_details, original_count, + len(filtered_units), library_mode=library_mode) + if _blackout: + dataset["metadata"]["reachability_filter"]["warning"] = _blackout + print(f" [Warning] {_blackout}", file=sys.stderr) + # Warn about unimplemented higher-level filters if processing_level == "codeql": print( @@ -374,7 +397,7 @@ def _load_module(name, filename): # Python parser # --------------------------------------------------------------------------- -def _parse_python(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None) -> ParseResult: +def _parse_python(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None, library_mode: bool = False) -> ParseResult: """Invoke the Python parser. The Python parser has a clean `parse_repository()` function that we can @@ -402,7 +425,8 @@ def _parse_python(repo_path: str, output_dir: str, processing_level: str, skip_t # Apply reachability filter if processing_level requires it if processing_level != "all": - dataset = _apply_reachability_filter(dataset, output_dir, processing_level) + dataset = _apply_reachability_filter(dataset, output_dir, processing_level, + library_mode=library_mode) # Write outputs write_json(dataset_path, dataset) @@ -523,7 +547,7 @@ def _file_lock(lock_path: Path): f.close() -def _parse_javascript(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None) -> ParseResult: +def _parse_javascript(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None, library_mode: bool = False) -> ParseResult: """Invoke the JavaScript/TypeScript parser. The JS parser is a PipelineTest class that runs Node.js subprocesses. @@ -547,6 +571,8 @@ def _parse_javascript(repo_path: str, output_dir: str, processing_level: str, sk cmd.extend(["--name", name]) if skip_tests: cmd.append("--skip-tests") + if library_mode: + cmd.append("--library-mode") result = subprocess.run( cmd, @@ -582,7 +608,7 @@ def _parse_javascript(repo_path: str, output_dir: str, processing_level: str, sk # Go parser # --------------------------------------------------------------------------- -def _parse_go(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None) -> ParseResult: +def _parse_go(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None, library_mode: bool = False) -> ParseResult: """Invoke the Go parser. The Go parser is a PipelineTest class that calls a compiled Go binary. @@ -603,6 +629,8 @@ def _parse_go(repo_path: str, output_dir: str, processing_level: str, skip_tests cmd.extend(["--name", name]) if skip_tests: cmd.append("--skip-tests") + if library_mode: + cmd.append("--library-mode") result = subprocess.run( cmd, @@ -638,7 +666,7 @@ def _parse_go(repo_path: str, output_dir: str, processing_level: str, skip_tests # C/C++ parser # --------------------------------------------------------------------------- -def _parse_c(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None) -> ParseResult: +def _parse_c(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None, library_mode: bool = False) -> ParseResult: """Invoke the C/C++ parser. The C parser uses tree-sitter for function extraction and call graph @@ -661,6 +689,8 @@ def _parse_c(repo_path: str, output_dir: str, processing_level: str, skip_tests: cmd.extend(["--name", name]) if skip_tests: cmd.append("--skip-tests") + if library_mode: + cmd.append("--library-mode") result = subprocess.run( cmd, @@ -697,7 +727,7 @@ def _parse_c(repo_path: str, output_dir: str, processing_level: str, skip_tests: # Ruby parser # --------------------------------------------------------------------------- -def _parse_ruby(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None) -> ParseResult: +def _parse_ruby(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None, library_mode: bool = False) -> ParseResult: """Invoke the Ruby parser. The Ruby parser uses tree-sitter for function extraction and call graph @@ -720,6 +750,8 @@ def _parse_ruby(repo_path: str, output_dir: str, processing_level: str, skip_tes cmd.extend(["--name", name]) if skip_tests: cmd.append("--skip-tests") + if library_mode: + cmd.append("--library-mode") result = subprocess.run( cmd, @@ -756,7 +788,7 @@ def _parse_ruby(repo_path: str, output_dir: str, processing_level: str, skip_tes # PHP parser # --------------------------------------------------------------------------- -def _parse_php(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None) -> ParseResult: +def _parse_php(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None, library_mode: bool = False) -> ParseResult: """Invoke the PHP parser. The PHP parser uses tree-sitter for function extraction and call graph @@ -779,6 +811,8 @@ def _parse_php(repo_path: str, output_dir: str, processing_level: str, skip_test cmd.extend(["--name", name]) if skip_tests: cmd.append("--skip-tests") + if library_mode: + cmd.append("--library-mode") result = subprocess.run( cmd, @@ -815,7 +849,7 @@ def _parse_php(repo_path: str, output_dir: str, processing_level: str, skip_test # Zig parser # --------------------------------------------------------------------------- -def _parse_zig(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None) -> ParseResult: +def _parse_zig(repo_path: str, output_dir: str, processing_level: str, skip_tests: bool = True, name: str = None, library_mode: bool = False) -> ParseResult: """Invoke the Zig parser. The Zig parser uses tree-sitter for function extraction and call graph @@ -838,6 +872,8 @@ def _parse_zig(repo_path: str, output_dir: str, processing_level: str, skip_test cmd.extend(["--name", name]) if skip_tests: cmd.append("--skip-tests") + if library_mode: + cmd.append("--library-mode") result = subprocess.run( cmd, diff --git a/libs/openant-core/core/scanner.py b/libs/openant-core/core/scanner.py index b17a98c5..4f6d4762 100644 --- a/libs/openant-core/core/scanner.py +++ b/libs/openant-core/core/scanner.py @@ -62,6 +62,7 @@ def scan_repository( diff_manifest: str | None = None, llm_reachability: bool = False, llm_reachability_max_code_bytes: int = 1500, + library_mode: bool = False, ) -> ScanResult: """Scan a repository for vulnerabilities. @@ -171,6 +172,7 @@ def _step_label(name: str) -> str: processing_level=effective_parse_level, skip_tests=skip_tests, diff_manifest=diff_manifest, + library_mode=library_mode, ) ctx.summary = { diff --git a/libs/openant-core/openant/cli.py b/libs/openant-core/openant/cli.py index b082d68a..686f34c0 100644 --- a/libs/openant-core/openant/cli.py +++ b/libs/openant-core/openant/cli.py @@ -75,6 +75,7 @@ def cmd_scan(args): repo_url=getattr(args, "repo_url", None), commit_sha=getattr(args, "commit_sha", None), diff_manifest=getattr(args, "diff_manifest", None), + library_mode=getattr(args, "library_mode", False), llm_reachability=getattr(args, "llm_reachability", False), llm_reachability_max_code_bytes=getattr( args, "llm_reachability_max_code_bytes", 1500 @@ -129,6 +130,7 @@ def cmd_parse(args): name=getattr(args, "name", None), diff_manifest=getattr(args, "diff_manifest", None), fresh=getattr(args, "fresh", False), + library_mode=getattr(args, "library_mode", False), ) ctx.summary = { @@ -972,7 +974,7 @@ def main(): scan_p.add_argument("--output", "-o", help="Output directory (default: temp dir)") scan_p.add_argument( "--language", "-l", - choices=["auto", "python", "javascript", "go", "c", "ruby", "php"], + choices=["auto", "python", "javascript", "go", "c", "ruby", "php", "zig"], default="auto", help="Language (default: auto-detect)", ) @@ -995,6 +997,8 @@ def main(): scan_p.add_argument("--dynamic-test", action="store_true", help="Enable Docker-isolated dynamic testing (off by default)") scan_p.add_argument("--no-skip-tests", action="store_true", help="Include test files in parsing (default: tests are skipped)") + scan_p.add_argument("--library-mode", action="store_true", + help="Seed the exported public API as entry points (for libraries with no main/route/CLI entry point)") scan_p.add_argument("--limit", type=int, help="Max units to analyze") scan_p.add_argument( "--llm-config", @@ -1047,7 +1051,7 @@ def main(): parse_p.add_argument("--output", "-o", help="Output directory (default: temp dir)") parse_p.add_argument( "--language", "-l", - choices=["auto", "python", "javascript", "go", "c", "ruby", "php"], + choices=["auto", "python", "javascript", "go", "c", "ruby", "php", "zig"], default="auto", help="Language (default: auto-detect)", ) @@ -1058,6 +1062,8 @@ def main(): help="Processing level (default: reachable)", ) parse_p.add_argument("--no-skip-tests", action="store_true", help="Include test files in parsing (default: tests are skipped)") + parse_p.add_argument("--library-mode", action="store_true", + help="Seed the exported public API as entry points (for libraries with no main/route/CLI entry point)") parse_p.add_argument("--name", help="Dataset name (default: derived from repo path)") parse_p.add_argument("--diff-manifest", help="Path to diff_manifest.json; tags units with diff_selected") parse_p.add_argument("--fresh", action="store_true", diff --git a/libs/openant-core/parsers/c/call_graph_builder.py b/libs/openant-core/parsers/c/call_graph_builder.py index 9de6faee..f5e92b7f 100644 --- a/libs/openant-core/parsers/c/call_graph_builder.py +++ b/libs/openant-core/parsers/c/call_graph_builder.py @@ -110,6 +110,11 @@ def __init__(self, extractor_output: Dict, options: Optional[Dict] = None): self.macros = extractor_output.get('macros', {}) self.macro_aliases = extractor_output.get('macro_aliases', {}) self.prototypes = extractor_output.get('prototypes', {}) + # class_name -> [direct base-class name, ...] for the inheritance walk in + # member dispatch (bug [30]). Defaults to {} when the extractor output + # predates base-class extraction, so resolution degrades to the [51] + # same-type behavior rather than erroring. + self.class_bases: Dict[str, List[str]] = extractor_output.get('class_bases', {}) self.repo_path = extractor_output.get('repository', '') self.max_depth = options.get('max_depth', 3) @@ -121,6 +126,10 @@ def __init__(self, extractor_output: Dict, options: Optional[Dict] = None): # Indexes for faster lookup self.functions_by_name: Dict[str, List[str]] = {} self.functions_by_file: Dict[str, List[str]] = {} + # class_name -> {base_method_name -> [func_id, ...]} for member dispatch. + # Scoped per (class, method) so a receiver-typed call resolves only to a + # method actually declared on that class, never a sibling/free function. + self.methods_by_class: Dict[str, Dict[str, List[str]]] = {} # Include map: file -> set of included header files self.include_map: Dict[str, Set[str]] = {} @@ -153,6 +162,13 @@ def _build_indexes(self) -> None: self.functions_by_file[file_path] = [] self.functions_by_file[file_path].append(func_id) + # Index methods by their declaring class for receiver-type dispatch. + class_name = func_data.get('class_name') + if class_name and name: + method_base = name.split('::')[-1] if '::' in name else name + self.methods_by_class.setdefault(class_name, {}) \ + .setdefault(method_base, []).append(func_id) + # Build include map for file_path, inc_list in self.includes.items(): self.include_map[file_path] = set() @@ -191,15 +207,25 @@ def _extract_calls_from_code(self, code: str, caller_id: str) -> Set[str]: except Exception: return self._extract_calls_regex(code, caller_id) + # Receiver static types inferred from local declarations in this body, + # used to resolve member calls (w.compute() / w->compute()) to the + # method on the receiver's known type. + local_var_types = self._extract_local_var_types(tree.root_node, code_bytes) + stack = [tree.root_node] while stack: node = stack.pop() if node.type == 'call_expression': func_node = node.child_by_field_name('function') if func_node: - call_name = self._extract_call_name(func_node, code_bytes) + call_name, receiver = self._extract_call_name_and_receiver( + func_node, code_bytes + ) if call_name: - resolved = self._resolve_call(call_name, caller_file) + receiver_type = local_var_types.get(receiver) if receiver else None + resolved = self._resolve_call(call_name, caller_file, + receiver_type=receiver_type, + is_member=func_node.type == 'field_expression') if resolved: calls.add(resolved) # A function passed by name as an argument (e.g. @@ -232,6 +258,81 @@ def _extract_callback_args(self, call_node, source: bytes, caller_file: str) -> found.add(resolved) return found + def _extract_call_name_and_receiver(self, node, source: bytes): + """Return (call_name, receiver_identifier) for a call's function child. + + receiver_identifier is the bare identifier text of a member-call receiver + (the `w` in `w.compute()` / `w->compute()`) when it is a simple + identifier, else None. The call_name is identical to what + _extract_call_name returns, so non-member calls are unaffected. + """ + if node.type == 'field_expression': + receiver = None + arg = node.child_by_field_name('argument') + if arg is not None and arg.type == 'identifier': + receiver = source[arg.start_byte:arg.end_byte].decode( + 'utf-8', errors='replace') + # _extract_call_name declines field_expression (no false free-function + # edges); the member name is recovered here from the `field` child and + # resolved ONLY through typed/same-file member dispatch in _resolve_call. + field = node.child_by_field_name('field') + name = None + if field is not None: + name = source[field.start_byte:field.end_byte].decode( + 'utf-8', errors='replace') + if not name.isidentifier(): + name = None + return name, receiver + return self._extract_call_name(node, source), None + + def _extract_local_var_types(self, root, source: bytes) -> Dict[str, str]: + """Map local variable name -> declared type name within a function body. + + Walks `declaration` nodes and records the (type_identifier, variable) + pairs for both plain declarations (`Widget w;`) and pointer declarations + (`Widget* w = ...;`). Only simple type_identifier types are recorded; + anything else (templates, qualified types, multiple declarators we can't + cleanly attribute) is skipped so callers fall back to base-name + resolution rather than risk a wrong-type edge. + """ + var_types: Dict[str, str] = {} + stack = [root] + while stack: + node = stack.pop() + if node.type == 'declaration': + type_node = node.child_by_field_name('type') + if type_node is not None and type_node.type == 'type_identifier': + type_name = source[type_node.start_byte:type_node.end_byte] \ + .decode('utf-8', errors='replace') + # A declaration can hold several declarators (Widget a, b;); + # attribute the type to every variable name we extract. + for child in node.children: + var_name = self._declared_var_name(child, source) + if var_name: + var_types[var_name] = type_name + stack.extend(reversed(node.children)) + return var_types + + def _declared_var_name(self, node, source: bytes) -> Optional[str]: + """Extract the declared variable identifier from a declarator subtree. + + Handles the plain identifier (`w`), pointer_declarator (`* w`) and + init_declarator (`* w = ...` / `w = ...`) shapes. Returns None for nodes + that are not a variable declarator (e.g. the type node, `;`). + """ + if node.type == 'identifier': + return source[node.start_byte:node.end_byte].decode('utf-8', errors='replace') + if node.type in ('pointer_declarator', 'init_declarator', 'reference_declarator'): + inner = node.child_by_field_name('declarator') + if inner is not None: + return self._declared_var_name(inner, source) + # init_declarator with no declarator field: scan children. + for child in node.children: + name = self._declared_var_name(child, source) + if name: + return name + return None + def _extract_call_name(self, node, source: bytes) -> Optional[str]: """Extract the function name from a call_expression's function child.""" text = source[node.start_byte:node.end_byte].decode('utf-8', errors='replace') @@ -274,27 +375,126 @@ def _is_visible_from(self, func_id: str, caller_file: str) -> bool: return True return not func_data.get('is_static', False) - def _resolve_call(self, call_name: str, caller_file: str) -> Optional[str]: - """Resolve a function call name to a function ID.""" + def _resolve_same_file(self, call_name: str, caller_file: str) -> Optional[str]: + """Resolve a call to a user-defined function in the same file, if any.""" + same_file_funcs = self.functions_by_file.get(caller_file, []) + for func_id in same_file_funcs: + func_data = self.functions.get(func_id, {}) + fname = func_data.get('name', '') + base_name = fname.split('::')[-1] if '::' in fname else fname + if base_name == call_name: + return func_id + return None + + def _resolve_method_on_class(self, class_name: str, call_name: str, + caller_file: str) -> Optional[str]: + """Resolve call_name to a method DIRECTLY declared on class_name (same file). + + Returns the func_id of a method named call_name declared on class_name and + defined in caller_file, else None. No inheritance — this is the single-hop + lookup the walk in _resolve_member_call composes over the base chain. + """ + by_method = self.methods_by_class.get(class_name) + if not by_method: + return None + for func_id in by_method.get(call_name, []): + func_data = self.functions.get(func_id, {}) + if func_data.get('file_path', '') == caller_file: + return func_id + return None + + def _resolve_member_call(self, call_name: str, caller_file: str, + receiver_type: str) -> Optional[str]: + """Resolve a member call to the method on the receiver's STATIC type, + walking UP the base-class chain to the first ancestor that defines it. + + Sound static-type floor (bug [30]): start at the receiver's declared type + and return its own method if it defines call_name; otherwise walk up its + direct base classes (BFS, cycle-guarded) and resolve to the FIRST ancestor + that declares call_name in the same file. The walk STOPS at the first + definer, so a derived override resolves to itself, never an ancestor. + + Deliberately does NOT link derived overrides of an ancestor's virtual + method (a documented non-goal that would create false edges): a call via a + Base* receiver resolves to Base's method only — the static-type floor. + + Same-file only: if no class on the chain defines call_name in this + translation unit, returns None so the caller falls back to base-name + resolution (never a wrong-type / unrelated-free-function edge). + """ + visited: Set[str] = set() + queue: List[str] = [receiver_type] + while queue: + cls = queue.pop(0) + if cls in visited: + continue + visited.add(cls) + # First definer on the chain wins (own type before ancestors). + match = self._resolve_method_on_class(cls, call_name, caller_file) + if match: + return match + for base in self.class_bases.get(cls, []): + if base not in visited: + queue.append(base) + return None + + def _resolve_call(self, call_name: str, caller_file: str, + receiver_type: Optional[str] = None, + is_member: bool = False, + _alias_chain: Optional[Set[str]] = None) -> Optional[str]: + """Resolve a function call name to a function ID. + + When receiver_type is given (a member call like w.compute() whose receiver + w has a known same-file type), resolve to that type's method FIRST. If + that fails, fall through to the unchanged base-name resolution below. + """ + if receiver_type: + member_match = self._resolve_member_call(call_name, caller_file, + receiver_type) + if member_match: + return member_match + + # A user-defined function in the SAME FILE shadows any stdlib/builtin + # of the same name, so it must be checked BEFORE the stdlib filter. + # Scope is deliberately same-file only: a genuine stdlib call (no + # same-file definition) still falls through to _is_stdlib below, so we + # never wrongly link a real stdlib call (e.g. printf/open) to an + # unrelated same-named user function in another file. + same_file_user_func = self._resolve_same_file(call_name, caller_file) + if same_file_user_func: + return same_file_user_func + + # A member call (obj->m() / obj.m()) whose receiver type is unknown or + # whose chain defines no such method resolves same-file only: declining + # here keeps the field-expression precision guarantee (never an edge to + # an unrelated cross-file free function of the same name). + if is_member: + return None + if self._is_stdlib(call_name): return None # Check for macro aliases resolved_name = self.macro_aliases.get(call_name, call_name) if resolved_name != call_name: - # Try resolving the aliased name instead - result = self._resolve_call(resolved_name, caller_file) - if result: - return result + # Guard against cyclic macro aliases (e.g. ``#define A B`` / + # ``#define B A`` -> {"A": "B", "B": "A"}). Without a visited-set + # the recursion below would loop A->B->A->... until RecursionError + # aborted the whole repo's call-graph build. + if _alias_chain is None: + _alias_chain = {call_name} + if resolved_name not in _alias_chain: + _alias_chain.add(resolved_name) + # Try resolving the aliased name instead + result = self._resolve_call(resolved_name, caller_file, + _alias_chain=_alias_chain) + if result: + return result # 1. Same-file functions - same_file_funcs = self.functions_by_file.get(caller_file, []) - for func_id in same_file_funcs: - func_data = self.functions.get(func_id, {}) - fname = func_data.get('name', '') - base_name = fname.split('::')[-1] if '::' in fname else fname - if base_name == call_name: - return func_id + same_file_match = self._resolve_same_file(call_name, caller_file) + if same_file_match: + return same_file_match # 2. Functions in included headers included_files = self.include_map.get(caller_file, set()) @@ -307,9 +507,18 @@ def _resolve_call(self, call_name: str, caller_file: str) -> Optional[str]: func_data = self.functions.get(func_id, {}) fname = func_data.get('name', '') base_name = fname.split('::')[-1] if '::' in fname else fname - # A static function in an included header has internal linkage - # and is not callable from the including file. - if base_name == call_name and self._is_visible_from(func_id, caller_file): + # A function in an INCLUDED header is callable from the including + # TU regardless of `static`: a `static inline` header helper (the + # C header-inline idiom) is copied into every includer. The + # cross-file static-linkage rejection in _is_visible_from is + # correct only for a .c TU, so it must not gate an included-header + # candidate. (Repo-wide and prototype fallbacks below stay strict.) + header_candidate = func_data.get('file_path', '').endswith( + ('.h', '.hpp', '.hxx', '.hh') + ) + if base_name == call_name and ( + header_candidate or self._is_visible_from(func_id, caller_file) + ): return func_id # 3. Unique name match across entire repo @@ -392,10 +601,13 @@ def _extract_calls_regex(self, code: str, caller_id: str) -> Set[str]: if func_name in ('if', 'while', 'for', 'switch', 'return', 'sizeof', 'typeof', 'alignof', 'offsetof', 'case', 'else'): continue - if not self._is_stdlib(func_name): - resolved = self._resolve_call(func_name, caller_file) - if resolved: - calls.add(resolved) + # No _is_stdlib gate here: _resolve_call applies the same-file-first + # rule and the stdlib filter internally, so a user function whose + # name collides with a builtin still resolves (same leak as the + # tree-sitter path otherwise). + resolved = self._resolve_call(func_name, caller_file) + if resolved: + calls.add(resolved) return calls diff --git a/libs/openant-core/parsers/c/function_extractor.py b/libs/openant-core/parsers/c/function_extractor.py index cae58601..198fdedd 100644 --- a/libs/openant-core/parsers/c/function_extractor.py +++ b/libs/openant-core/parsers/c/function_extractor.py @@ -66,6 +66,10 @@ def __init__(self, repo_path: str): self.macros: Dict[str, List[Dict]] = {} self.macro_aliases: Dict[str, str] = {} # e.g. OPENSSL_malloc -> CRYPTO_malloc self.prototypes: Dict[str, Dict] = {} # function name -> declaration info + # class/struct name -> list of direct base-class names, for inheritance + # walks in member dispatch (bug [30]). Populated from the + # base_class_clause of each class_specifier/struct_specifier. + self.class_bases: Dict[str, List[str]] = {} self.c_parser = Parser(C_LANGUAGE) self.cpp_parser = Parser(CPP_LANGUAGE) @@ -148,11 +152,18 @@ def _extract_identifier_from_declarator(self, node, source: bytes) -> Optional[s if node.type == 'qualified_identifier': return self._node_text(node, source) - # template_function + # operator overload (operator+, operator==, operator[], ...) + if node.type == 'operator_name': + return self._node_text(node, source) + + # conversion operator (operator int, operator MyType) + if node.type == 'operator_cast': + return self._node_text(node, source) + + # template_function: g — keep the template arguments so an + # explicit specialization does not collide with the primary template. if node.type == 'template_function': - name_node = node.child_by_field_name('name') - if name_node: - return self._extract_identifier_from_declarator(name_node, source) + return self._node_text(node, source) # reference_declarator (C++ int& func()) if node.type == 'reference_declarator': @@ -170,7 +181,8 @@ def _extract_identifier_from_declarator(self, node, source: bytes) -> Optional[s if child.type in ('identifier', 'field_identifier', 'qualified_identifier', 'function_declarator', 'pointer_declarator', 'parenthesized_declarator', 'destructor_name', - 'template_function', 'reference_declarator'): + 'template_function', 'reference_declarator', + 'operator_name', 'operator_cast'): result = self._extract_identifier_from_declarator(child, source) if result: return result @@ -218,6 +230,62 @@ def _get_parameters(self, node, source: bytes) -> List[str]: params.append('...') return params + @staticmethod + def _signature_discriminator(parameters: List[str]) -> str: + """A deterministic, COLON-FREE signature string for the parameter list. + + Colon-free is mandatory: the func_id is split on ':' downstream + (`core/diff_filter.py`, `utilities/agentic_enhancer/repository_index.py` + rsplit on ':' to recover the file), so any colon a C++ type carries + (`std::string`, a ternary default arg) is mapped to '.' here. + """ + joined = ','.join(p.strip() for p in parameters) + joined = ' '.join(joined.split()) # collapse newlines/runs of whitespace + return joined.replace(':', '.') + + @staticmethod + def _same_signature(a: dict, b: dict) -> bool: + """Two definitions share a signature iff their parameter lists match. + + Identical params => a redefinition of the SAME function under a different + preprocessor branch (#ifdef/#else) or an ODR-duplicate. Different params + => a genuine C++ overload. + """ + return a.get('parameters', []) == b.get('parameters', []) + + def _store_function(self, func_id: str, func_data: dict) -> None: + """Store a function, disambiguating same-(file,name) collisions instead + of silently overwriting (Bug 1: #ifdef/#else stubs and C++ overloads + collapsed onto one node — see reachability-bugs.md / bug-3482.md). + + Collision-only: when func_id is free, store byte-identically (the + hardcoded `path:name` id literals across the suite depend on unique names + keeping the plain `path:name` id). On a collision: + - SAME signature -> keep ONE node, prefer the larger body. The #else + stub is shorter than the real implementation regardless of which + preprocessor arm tree-sitter emits first, so this is order- and + direction-independent. + - DIFFERENT signature -> a genuine overload; both are real. Mint a + distinct key by folding the colon-free signature into the func_id. + The `name` field is left bare (`area`, not `area(int,int)`) because + the call-graph builder resolves calls by `name` (functions_by_name), + so both overloads stay findable. (Contrast bug [39], which folds the + template-arg discriminator into the NAME — correct there because a + template call is written `g`; an overload call is written bare.) + """ + existing = self.functions.get(func_id) + if existing is None: + self.functions[func_id] = func_data + return + if self._same_signature(existing, func_data): + if len(func_data.get('code', '')) > len(existing.get('code', '')): + self.functions[func_id] = func_data + return + overload_id = f"{func_id}({self._signature_discriminator(func_data.get('parameters', []))})" + prior = self.functions.get(overload_id) + if prior is None or len(func_data.get('code', '')) > len(prior.get('code', '')): + self.functions[overload_id] = func_data + def _find_function_declarator(self, node): """Find the function_declarator within a declarator tree.""" if node.type == 'function_declarator': @@ -257,10 +325,15 @@ def _classify_function(self, name: str, file_path: str, is_static: bool, return 'main' if is_cpp and class_name: - if name == class_name: - return 'constructor' - if name.startswith('~'): + # Compare the UNqualified leaves: an out-of-line definition arrives + # as a qualified name (e.g. 'Foo::Foo', 'Foo::~Foo') whose whole + # string never equals the bare class_name. + unqualified = name.rsplit('::', 1)[-1] + class_leaf = class_name.rsplit('::', 1)[-1] + if unqualified.startswith('~'): return 'destructor' + if unqualified == class_leaf: + return 'constructor' return 'method' if '__attribute__((constructor))' in code: @@ -289,6 +362,27 @@ def _get_class_name_from_qualified(self, name: str) -> Optional[str]: return '::'.join(parts[:-1]) return None + def _extract_base_classes(self, record_node, source: bytes) -> List[str]: + """Return the direct base-class names of a class/struct specifier. + + tree-sitter represents inheritance as a `base_class_clause` child of the + class_specifier/struct_specifier (sibling of name and body). The clause + holds one `type_identifier` per base, interleaved with optional + access_specifier / `virtual` / `,` tokens; we collect only the simple + `type_identifier` bases. Qualified or templated bases (ns::Base, + Base) are NOT recorded — matching the same-name keying used for + class_name elsewhere, so the inheritance walk stays sound (an unknown + base simply yields no edge rather than a wrong one). + """ + bases: List[str] = [] + for child in record_node.children: + if child.type != 'base_class_clause': + continue + for sub in child.children: + if sub.type == 'type_identifier': + bases.append(self._node_text(sub, source)) + return bases + def _extract_includes(self, tree, source: bytes) -> List[str]: """Extract #include directives from a file.""" includes = [] @@ -355,23 +449,38 @@ def _extract_functions_from_tree(self, tree, source: bytes, file_path: str, """Extract all function definitions from a parsed tree.""" is_header = os.path.splitext(file_path)[1].lower() in ('.h', '.hpp', '.hxx', '.hh') - # Iterative traversal with explicit stack carrying (node, namespace_prefix) - stack = [(tree.root_node, '')] + # Iterative traversal with explicit stack carrying + # (node, namespace_prefix, class_context). namespace_prefix is the + # qualified prefix used to build the func_id; class_context is the + # enclosing class/struct/union name (or None) used for metadata so a + # namespace qualifier is never mistaken for a class qualifier. + stack = [(tree.root_node, '', None)] + + # struct/union member functions are C++ methods exactly like class + # members; only `class_specifier` was special-cased originally. + record_specifiers = ('class_specifier', 'struct_specifier', 'union_specifier') while stack: - node, namespace_prefix = stack.pop() + node, namespace_prefix, class_context = stack.pop() if node.type == 'function_definition': self._process_function_node(node, source, relative_path, - is_cpp, is_header, namespace_prefix) + is_cpp, is_header, namespace_prefix, + class_context) elif node.type == 'declaration' and not is_header: - # Skip standalone declarations in .c files (prototypes only) - pass + # Standalone declarations in .c/.cpp files are prototypes, EXCEPT + # a declaration whose initializer is a lambda — a named callable. + if is_cpp: + self._process_lambda_declaration(node, source, relative_path, + namespace_prefix) elif node.type == 'declaration' and is_header: # In headers, track prototypes for call resolution self._process_declaration_node(node, source, relative_path) + if is_cpp: + self._process_lambda_declaration(node, source, relative_path, + namespace_prefix) elif node.type == 'namespace_definition' and is_cpp: ns_name_node = node.child_by_field_name('name') @@ -380,42 +489,62 @@ def _extract_functions_from_tree(self, tree, source: bytes, file_path: str, body_node = node.child_by_field_name('body') if body_node: for child in reversed(body_node.children): - stack.append((child, new_prefix)) + # class_context unchanged: a namespace is NOT a class. + stack.append((child, new_prefix, class_context)) continue # Don't walk children again - elif node.type == 'class_specifier' and is_cpp: + elif node.type in record_specifiers and is_cpp: class_name_node = node.child_by_field_name('name') if class_name_node: class_name = self._node_text(class_name_node, source) new_prefix = f"{namespace_prefix}{class_name}::" + # Record direct base classes for the inheritance walk in + # member dispatch (bug [30]). Keyed by the bare class name + # (matching the class_name stored on each method). + bases = self._extract_base_classes(node, source) + if bases: + existing = self.class_bases.setdefault(class_name, []) + for base in bases: + if base not in existing: + existing.append(base) body_node = node.child_by_field_name('body') if body_node: for child in reversed(body_node.children): if child.type == 'function_definition': self._process_function_node( child, source, relative_path, - is_cpp, is_header, new_prefix + is_cpp, is_header, new_prefix, class_name ) elif child.type == 'access_specifier': pass else: - stack.append((child, new_prefix)) + stack.append((child, new_prefix, class_name)) continue else: for child in reversed(node.children): - stack.append((child, namespace_prefix)) + stack.append((child, namespace_prefix, class_context)) def _process_function_node(self, node, source: bytes, relative_path: str, is_cpp: bool, is_header: bool, - namespace_prefix: str = '') -> None: + namespace_prefix: str = '', + class_context: Optional[str] = None) -> None: """Process a single function_definition node.""" name = self._get_function_name(node, source) if not name: return full_name = namespace_prefix + name if namespace_prefix and '::' not in name else name - class_name = self._get_class_name_from_qualified(full_name) + # class_name comes from the lexical enclosing class/struct/union + # (class_context) OR from a qualifier written in the source name + # (out-of-line def: Foo::method). A namespace qualifier in + # namespace_prefix must NOT become a class_name. + if class_context is not None: + class_name = class_context + elif '::' in name: + class_name = self._get_class_name_from_qualified(name) + else: + class_name = None code = self._node_text(node, source) start_line = node.start_point[0] + 1 # tree-sitter is 0-indexed @@ -447,7 +576,7 @@ def _process_function_node(self, node, source: bytes, relative_path: str, 'class_name': class_name, } - self.functions[func_id] = func_data + self._store_function(func_id, func_data) self.stats['total_functions'] += 1 if is_static: @@ -457,6 +586,52 @@ def _process_function_node(self, node, source: bytes, relative_path: str, self.stats['by_type'][unit_type] = self.stats['by_type'].get(unit_type, 0) + 1 + def _process_lambda_declaration(self, node, source: bytes, relative_path: str, + namespace_prefix: str = '') -> None: + """Extract a named lambda from a declaration (auto f = [](){...};). + + A lambda is a `lambda_expression` initializer inside an + `init_declarator`; the declaration node is otherwise skipped by the + traversal, so the callable would never be recorded as a unit. + """ + # A declaration may declare several init_declarators. + stack = list(node.children) + while stack: + child = stack.pop() + if child.type != 'init_declarator': + stack.extend(child.children) + continue + + value_node = child.child_by_field_name('value') + if value_node is None or value_node.type != 'lambda_expression': + continue + + decl_node = child.child_by_field_name('declarator') + name = (self._extract_identifier_from_declarator(decl_node, source) + if decl_node is not None else None) + if not name: + continue + + full_name = (namespace_prefix + name + if namespace_prefix and '::' not in name else name) + func_id = f"{relative_path}:{full_name}" + self._store_function(func_id, { + 'name': full_name, + 'file_path': relative_path, + 'start_line': node.start_point[0] + 1, + 'end_line': node.end_point[0] + 1, + 'code': self._node_text(node, source), + 'parameters': self._get_parameters(value_node, source), + 'return_type': 'auto', + 'is_static': False, + 'is_exported': False, + 'is_inline': False, + 'unit_type': 'lambda', + 'class_name': None, + }) + self.stats['total_functions'] += 1 + self.stats['by_type']['lambda'] = self.stats['by_type'].get('lambda', 0) + 1 + def _process_declaration_node(self, node, source: bytes, relative_path: str) -> None: """Process a declaration node in a header to track prototypes.""" # Look for function declarations (prototypes) @@ -486,7 +661,7 @@ def process_file(self, file_path: Path) -> None: self.stats['files_with_errors'] += 1 return - relative_path = str(file_path.relative_to(self.repo_path)) + relative_path = file_path.relative_to(self.repo_path).as_posix() is_cpp = self._is_cpp_file(str(file_path)) parser = self._get_parser(str(file_path)) @@ -550,6 +725,7 @@ def export(self) -> Dict: 'macros': self.macros, 'macro_aliases': self.macro_aliases, 'prototypes': self.prototypes, + 'class_bases': self.class_bases, 'statistics': self.stats, } diff --git a/libs/openant-core/parsers/c/test_pipeline.py b/libs/openant-core/parsers/c/test_pipeline.py index c19824db..883b8bf4 100644 --- a/libs/openant-core/parsers/c/test_pipeline.py +++ b/libs/openant-core/parsers/c/test_pipeline.py @@ -48,7 +48,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) from utilities.file_io import open_utf8, read_json, run_utf8, write_json from utilities.context_enhancer import ContextEnhancer -from utilities.agentic_enhancer import EntryPointDetector, ReachabilityAnalyzer +from utilities.agentic_enhancer import EntryPointDetector, ReachabilityAnalyzer, blackout_warning, library_seed_ids # Local imports from repository_scanner import RepositoryScanner @@ -87,7 +87,8 @@ def __init__( processing_level: ProcessingLevel = ProcessingLevel.ALL, skip_tests: bool = False, depth: int = 3, - name: str = None + name: str = None, + library_mode: bool = False ): self.repo_path = os.path.abspath(repo_path) self.output_dir = output_dir or os.path.join(os.path.dirname(__file__), 'test_output') @@ -98,6 +99,7 @@ def __init__( self.skip_tests = skip_tests self.depth = depth self.dataset_name = name + self.library_mode = library_mode # Pipeline artifacts self.scan_results_file = None @@ -292,6 +294,13 @@ def apply_reachability_filter(self) -> bool: detector = EntryPointDetector(normalized_functions, call_graph) self.entry_points = detector.detect_entry_points() + # Library-mode: a library's entry surface is its exported public API, + # which carries no main/route/CLI marker. Seed it so the BFS reaches + # the core instead of blacking out. Union-only — never drops a + # structurally-detected entry point. + if self.library_mode: + self.entry_points = self.entry_points | library_seed_ids(normalized_functions) + # Build reachability reachability = ReachabilityAnalyzer( functions=normalized_functions, @@ -323,6 +332,13 @@ def apply_reachability_filter(self) -> bool: "reduction_percentage": round((1 - len(filtered_units) / original_count) * 100, 1) if original_count > 0 else 0 } + _blackout = blackout_warning(detector.entry_point_details, original_count, + len(filtered_units), + library_mode=getattr(self, "library_mode", False)) + if _blackout: + dataset["metadata"]["reachability_filter"]["warning"] = _blackout + print(f" [Warning] {_blackout}", file=sys.stderr) + write_json(self.dataset_file, dataset) elapsed = (datetime.now() - start_time).total_seconds() @@ -1027,6 +1043,11 @@ def main(): default=None, help='Dataset name (default: derived from repo path)' ) + parser.add_argument( + '--library-mode', + action='store_true', + help='Seed the exported public API as entry points (for libraries with no main/route/CLI)' + ) args = parser.parse_args() @@ -1048,7 +1069,8 @@ def main(): processing_level=processing_level, skip_tests=args.skip_tests, depth=args.depth, - name=args.name + name=args.name, + library_mode=args.library_mode ) results = pipeline.run_full_pipeline() diff --git a/libs/openant-core/parsers/c/unit_generator.py b/libs/openant-core/parsers/c/unit_generator.py index fcca5065..455bd165 100644 --- a/libs/openant-core/parsers/c/unit_generator.py +++ b/libs/openant-core/parsers/c/unit_generator.py @@ -237,6 +237,7 @@ def create_unit(self, func_id: str, func_data: Dict) -> Dict: 'metadata': { 'is_static': func_data.get('is_static', False), 'is_exported': func_data.get('is_exported', True), + 'is_inline': func_data.get('is_inline', False), 'return_type': func_data.get('return_type', ''), 'parameters': func_data.get('parameters', []), 'generator': 'c_unit_generator.py', @@ -306,6 +307,7 @@ def generate_analyzer_output(self) -> Dict: 'endLine': func_data.get('end_line', 0), 'isStatic': func_data.get('is_static', False), 'isExported': func_data.get('is_exported', True), + 'isInline': func_data.get('is_inline', False), 'returnType': func_data.get('return_type', ''), 'parameters': func_data.get('parameters', []), 'className': func_data.get('class_name'), diff --git a/libs/openant-core/parsers/go/go_parser/callgraph.go b/libs/openant-core/parsers/go/go_parser/callgraph.go index 5ff47f12..7d848f4d 100644 --- a/libs/openant-core/parsers/go/go_parser/callgraph.go +++ b/libs/openant-core/parsers/go/go_parser/callgraph.go @@ -200,6 +200,13 @@ func (c *CallGraphBuilder) extractCalls(funcInfo FunctionInfo) []CallInfo { // import table instead of a name-shape heuristic. imports := c.importsByFile[funcInfo.FilePath] + // Track simple func-value aliases (f := helper) so a later call f() + // resolves to the aliased function. Only single, unconditional bindings + // of the form `name := ` / `name = ` are tracked; any + // reassignment (or a non-ident RHS) marks the name ambiguous so we emit + // no false edge — precision over recall. + aliases := c.collectFuncValueAliases(file) + // Walk the AST looking for call expressions ast.Inspect(file, func(n ast.Node) bool { call, ok := n.(*ast.CallExpr) @@ -208,6 +215,13 @@ func (c *CallGraphBuilder) extractCalls(funcInfo FunctionInfo) []CallInfo { } callInfo := c.analyzeCallExpr(call, imports) + // Rewrite an unambiguous func-value alias call (f()) to its target + // (helper()) so it resolves like a direct call. + if callInfo.Name != "" && callInfo.Receiver == "" && callInfo.Package == "" { + if target, ok := aliases[callInfo.Name]; ok { + callInfo.Name = target + } + } if callInfo.Name != "" && !c.builtins[callInfo.Name] && !c.builtins[callInfo.Package] { calls = append(calls, callInfo) } @@ -217,20 +231,88 @@ func (c *CallGraphBuilder) extractCalls(funcInfo FunctionInfo) []CallInfo { return calls } +// collectFuncValueAliases scans a parsed function body for single, unconditional +// func-value bindings (`f := helper`) and returns name -> target-function-name. +// A name bound more than once, or bound to anything other than a bare identifier, +// is dropped (left out of the map) so a reassigned/conditional alias never +// produces a false edge. +func (c *CallGraphBuilder) collectFuncValueAliases(file *ast.File) map[string]string { + aliases := make(map[string]string) + ambiguous := make(map[string]bool) + + record := func(lhs, rhs ast.Expr) { + lid, ok := lhs.(*ast.Ident) + if !ok { + return + } + if ambiguous[lid.Name] { + return + } + rid, ok := rhs.(*ast.Ident) + if !ok { + // Bound to a non-ident (call, selector, literal, ...) -> ambiguous. + delete(aliases, lid.Name) + ambiguous[lid.Name] = true + return + } + if _, seen := aliases[lid.Name]; seen { + // Second binding of the same name -> ambiguous, drop it. + delete(aliases, lid.Name) + ambiguous[lid.Name] = true + return + } + aliases[lid.Name] = rid.Name + } + + ast.Inspect(file, func(n ast.Node) bool { + assign, ok := n.(*ast.AssignStmt) + if !ok { + return true + } + // Only handle 1:1 bindings (f := helper); skip tuple assignments. + if len(assign.Lhs) != 1 || len(assign.Rhs) != 1 { + // Mark any ident LHS ambiguous so a multi-value rebind can't alias. + for _, lhs := range assign.Lhs { + if lid, ok := lhs.(*ast.Ident); ok { + delete(aliases, lid.Name) + ambiguous[lid.Name] = true + } + } + return true + } + record(assign.Lhs[0], assign.Rhs[0]) + return true + }) + + return aliases +} + func (c *CallGraphBuilder) analyzeCallExpr(call *ast.CallExpr, imports map[string]string) CallInfo { info := CallInfo{} - switch fun := call.Fun.(type) { + // Unwrap a generic instantiation so fn[T](), fn[K,V](), obj.M[T]() and obj.M[K,V]() are + // analyzed identically to their non-generic forms. A single type argument parses as + // *ast.IndexExpr, multiple as *ast.IndexListExpr; both wrap the underlying function + // expression (an Ident or a SelectorExpr) in .X. + fun := call.Fun + switch idx := fun.(type) { + case *ast.IndexExpr: + fun = idx.X + case *ast.IndexListExpr: + fun = idx.X + } + + switch f := fun.(type) { case *ast.Ident: - // Simple call: funcName() - info.Name = fun.Name + // Simple call: funcName() (or generic Gen[..]()) + info.Name = f.Name case *ast.SelectorExpr: - // Method or package call: obj.Method() or pkg.Func() - info.Name = fun.Sel.Name + // Method or package call: obj.Method() or pkg.Func() (or generic obj.M[..]()) + info.Name = f.Sel.Name info.IsMethod = true - switch x := fun.X.(type) { + switch x := f.X.(type) { case *ast.Ident: info.Receiver = x.Name // It is a package call iff the receiver name is an import alias of this file. @@ -248,12 +330,6 @@ func (c *CallGraphBuilder) analyzeCallExpr(call *ast.CallExpr, imports map[strin // Result of another call: getObj().Method() info.Receiver = "~call_result~" } - - case *ast.IndexExpr: - // Generic function call: fn[T]() - if ident, ok := fun.X.(*ast.Ident); ok { - info.Name = ident.Name - } } return info diff --git a/libs/openant-core/parsers/go/go_parser/callgraph_funcvalue_test.go b/libs/openant-core/parsers/go/go_parser/callgraph_funcvalue_test.go new file mode 100644 index 00000000..3bb1813f --- /dev/null +++ b/libs/openant-core/parsers/go/go_parser/callgraph_funcvalue_test.go @@ -0,0 +1,86 @@ +package main + +// Regression test for BUG-NEW-2026-06-04-go-dataflow_loss: a call through a +// function-value alias (f := helper; f()) must emit an edge caller -> helper, +// mirroring the direct call helper(). + +import "testing" + +// buildGraphForFuncs runs the full call-graph build over a synthetic +// AnalyzerOutput (same shape the extractor emits) and returns the call graph. +func buildGraphForFuncs(t *testing.T, funcs map[string]FunctionInfo) map[string][]string { + t.Helper() + builder := NewCallGraphBuilder(".") + analyzer := &AnalyzerOutput{RepoRoot: ".", Functions: funcs} + // Index directly (parseImports reads real files; none of these synthetic + // funcs use package-qualified calls, so an empty import table is fine). + for funcID, funcInfo := range analyzer.Functions { + builder.functionsByName[funcInfo.Name] = append(builder.functionsByName[funcInfo.Name], funcID) + builder.functionsByFile[funcInfo.FilePath] = append(builder.functionsByFile[funcInfo.FilePath], funcID) + if funcInfo.ClassName != "" { + builder.methodsByType[funcInfo.ClassName] = append(builder.methodsByType[funcInfo.ClassName], funcID) + } + } + cg := make(map[string][]string) + for funcID, funcInfo := range analyzer.Functions { + calls := builder.extractCalls(funcInfo) + resolved := builder.resolveCalls(funcID, funcInfo, calls, analyzer) + if len(resolved) > 0 { + cg[funcID] = resolved + } + } + return cg +} + +func hasEdge(cg map[string][]string, from, to string) bool { + for _, t := range cg[from] { + if t == to { + return true + } + } + return false +} + +// Baseline control: a direct call helper() resolves to an edge. +func TestDirectCallEdge(t *testing.T) { + funcs := map[string]FunctionInfo{ + "main.go:helper": {Name: "helper", FilePath: "main.go", Package: "main", + Code: "func helper() int {\n\treturn 42\n}"}, + "main.go:caller": {Name: "caller", FilePath: "main.go", Package: "main", + Code: "func caller() int {\n\treturn helper()\n}"}, + } + cg := buildGraphForFuncs(t, funcs) + if !hasEdge(cg, "main.go:caller", "main.go:helper") { + t.Fatalf("baseline: expected edge main.go:caller -> main.go:helper, got %v", cg) + } +} + +// BUG-19: a func-value alias f := helper; f() must resolve to the same edge. +func TestFuncValueAliasEdge(t *testing.T) { + funcs := map[string]FunctionInfo{ + "main.go:helper": {Name: "helper", FilePath: "main.go", Package: "main", + Code: "func helper() int {\n\treturn 42\n}"}, + "main.go:caller": {Name: "caller", FilePath: "main.go", Package: "main", + Code: "func caller() int {\n\tf := helper\n\treturn f()\n}"}, + } + cg := buildGraphForFuncs(t, funcs) + if !hasEdge(cg, "main.go:caller", "main.go:helper") { + t.Fatalf("alias: expected edge main.go:caller -> main.go:helper, got %v", cg) + } +} + +// Precision guard: a reassigned/conditional alias must NOT produce a false edge. +func TestFuncValueAliasReassignedNoEdge(t *testing.T) { + funcs := map[string]FunctionInfo{ + "main.go:helper": {Name: "helper", FilePath: "main.go", Package: "main", + Code: "func helper() int {\n\treturn 42\n}"}, + "main.go:other": {Name: "other", FilePath: "main.go", Package: "main", + Code: "func other() int {\n\treturn 7\n}"}, + "main.go:caller": {Name: "caller", FilePath: "main.go", Package: "main", + Code: "func caller() int {\n\tf := helper\n\tf = other\n\treturn f()\n}"}, + } + cg := buildGraphForFuncs(t, funcs) + if hasEdge(cg, "main.go:caller", "main.go:helper") { + t.Fatalf("reassigned: must NOT resolve f() to helper after f=other, got %v", cg) + } +} diff --git a/libs/openant-core/parsers/go/go_parser/extractor.go b/libs/openant-core/parsers/go/go_parser/extractor.go index 5f3a1d13..6c60b342 100644 --- a/libs/openant-core/parsers/go/go_parser/extractor.go +++ b/libs/openant-core/parsers/go/go_parser/extractor.go @@ -72,6 +72,9 @@ func (e *Extractor) extractFromFile(filePath string, output *AnalyzerOutput) err case *ast.FuncDecl: funcInfo := e.extractFunctionDecl(d, relPath, pkgName, content) funcID := e.makeFunctionID(relPath, funcInfo) + if w := duplicateIDWarning(output.Functions, funcID, funcInfo); w != "" { + fmt.Fprintln(os.Stderr, "Warning: "+w) + } output.Functions[funcID] = funcInfo } } @@ -186,6 +189,14 @@ func (e *Extractor) typeToString(expr ast.Expr) string { return "*" + e.typeToString(t.X) case *ast.SelectorExpr: return e.typeToString(t.X) + "." + t.Sel.Name + case *ast.IndexExpr: + // Generic type with one type parameter, e.g. Stack[T]. The class key is the + // base type, so drop the type argument and recurse into the base (t.X). + return e.typeToString(t.X) + case *ast.IndexListExpr: + // Generic type with multiple type parameters, e.g. Pair[K, V]. Same as above: + // the class key is the bare base type, so recurse into t.X and drop the args. + return e.typeToString(t.X) case *ast.ArrayType: if t.Len == nil { return "[]" + e.typeToString(t.Elt) @@ -273,11 +284,13 @@ var httpHandlerPatterns = []*regexp.Regexp{ func (e *Extractor) isHTTPHandler(params, returns []string, code string) bool { paramsStr := strings.Join(params, " ") - returnsStr := strings.Join(returns, " ") - // Check parameters for HTTP patterns + // Check parameters for HTTP patterns. A handler is identified by what it + // RECEIVES (request/context types in params), not by what it returns — + // matching these patterns against the return type mis-tagged factories + // such as `func Logger() gin.HandlerFunc` as http_handler entry points (F10). for _, pattern := range httpHandlerPatterns { - if pattern.MatchString(paramsStr) || pattern.MatchString(returnsStr) { + if pattern.MatchString(paramsStr) { return true } } @@ -325,17 +338,41 @@ func (e *Extractor) isCLIHandler(params, returns []string, code string) bool { func (e *Extractor) isMiddleware(params, returns []string, code string) bool { // Middleware often takes and returns http.Handler or similar returnsStr := strings.Join(returns, " ") + paramsStr := strings.Join(params, " ") + + // A bare `next(` substring previously matched any body with a next() call + // (Go 1.23 iterators, lexers, cursors), mislabelling them middleware (F7). + // Only treat a `next(` call as middleware when the function has an HTTP + // request signature, i.e. it is actually wrapping a handler. + hasHTTPSignature := strings.Contains(paramsStr, "http.ResponseWriter") || + strings.Contains(paramsStr, "http.Request") || + strings.Contains(paramsStr, "http.Handler") if strings.Contains(returnsStr, "http.Handler") || strings.Contains(returnsStr, "http.HandlerFunc") || strings.Contains(code, "next.ServeHTTP") || - strings.Contains(code, "next(") { + (hasHTTPSignature && strings.Contains(code, "next(")) { return true } return false } +// duplicateIDWarning returns a non-empty diagnostic when funcID already maps to a unit in fns. +// A collision means two distinct declarations resolved to the same unit id — typically a +// class-key collapse (e.g. an unhandled receiver shape falling back to "unknown") — and the +// subsequent `fns[funcID] = ...` write would silently overwrite the earlier unit (data loss). +// The caller logs the returned string so the collapse is observable instead of silent. An empty +// string means no collision (the common case). +func duplicateIDWarning(fns map[string]FunctionInfo, funcID string, incoming FunctionInfo) string { + existing, ok := fns[funcID] + if !ok { + return "" + } + return fmt.Sprintf("duplicate unit id %q: %s:%d would overwrite %s:%d (class-key collapse?)", + funcID, incoming.FilePath, incoming.StartLine, existing.FilePath, existing.StartLine) +} + func (e *Extractor) makeFunctionID(filePath string, info FunctionInfo) string { if info.ClassName != "" { return fmt.Sprintf("%s:%s.%s", filePath, info.ClassName, info.Name) diff --git a/libs/openant-core/parsers/go/go_parser/extractor_classify_test.go b/libs/openant-core/parsers/go/go_parser/extractor_classify_test.go new file mode 100644 index 00000000..1d112e51 --- /dev/null +++ b/libs/openant-core/parsers/go/go_parser/extractor_classify_test.go @@ -0,0 +1,70 @@ +package main + +import "testing" + +// Regression tests for F7 and F10 — over-broad unit-type classification in +// classifyUnitType, which seeds false remote-web entry points downstream. + +// F10: a factory that *returns* http.HandlerFunc (no request-shaped params, +// no handler body) must NOT be classified http_handler — only a function that +// RECEIVES a request is a handler. +func TestFactoryReturningHandlerFuncIsNotHTTPHandler(t *testing.T) { + e := NewExtractor(".") + got := e.classifyUnitType( + "NewLoggingHandler", "", + []string{"prefix string"}, // params: no request types + []string{"http.HandlerFunc"}, // returns a handler (factory) + "{ return func(w http.ResponseWriter, r *http.Request) {} }", + "x.go", + ) + if got == UnitTypeHTTPHandler { + t.Fatalf("factory returning http.HandlerFunc classified as %q, want != http_handler", got) + } +} + +// F7: an iterator/lexer whose body merely contains a "next(" call must NOT be +// classified middleware. Real middleware returns http.Handler/HandlerFunc or +// calls next.ServeHTTP / next(w, r) with an http signature. +func TestIteratorWithNextCallIsNotMiddleware(t *testing.T) { + e := NewExtractor(".") + got := e.classifyUnitType( + "CutPrefix", "", + []string{"s string", "prefix string"}, // no http params + []string{"string", "bool"}, // no http returns + "{ for next, hasNext := iter(); hasNext; { _ = next(); } ; return s, true }", + "seq.go", + ) + if got == UnitTypeMiddleware { + t.Fatalf("iterator using next() classified as %q, want != middleware", got) + } +} + +// Guard against over-correction: a genuine net/http handler and genuine +// middleware must still be detected. +func TestGenuineHTTPHandlerStillDetected(t *testing.T) { + e := NewExtractor(".") + got := e.classifyUnitType( + "ServeIndex", "", + []string{"w http.ResponseWriter", "r *http.Request"}, + []string{}, + "{ w.Write([]byte(\"ok\")) }", + "h.go", + ) + if got != UnitTypeHTTPHandler { + t.Fatalf("genuine handler classified as %q, want http_handler", got) + } +} + +func TestGenuineMiddlewareStillDetected(t *testing.T) { + e := NewExtractor(".") + got := e.classifyUnitType( + "Logging", "", + []string{"next http.Handler"}, + []string{"http.Handler"}, + "{ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request){ next.ServeHTTP(w, r) }) }", + "mw.go", + ) + if got != UnitTypeMiddleware { + t.Fatalf("genuine middleware classified as %q, want middleware", got) + } +} diff --git a/libs/openant-core/parsers/go/go_parser/extractor_dupguard_test.go b/libs/openant-core/parsers/go/go_parser/extractor_dupguard_test.go new file mode 100644 index 00000000..2f3ecdde --- /dev/null +++ b/libs/openant-core/parsers/go/go_parser/extractor_dupguard_test.go @@ -0,0 +1,39 @@ +package main + +// Guard test for the duplicate-unit-id store hazard (substrate defense-in-depth). +// +// The extractor stores units as `output.Functions[funcID] = info`. If two distinct declarations +// resolve to the same funcID — the failure mode behind the generic-receiver class-key collapse, +// where many methods became "m.go:unknown.Name" — the second write silently overwrites the first +// and a real unit vanishes with no signal. duplicateIDWarning makes that collision observable +// (the caller logs it to stderr) so a future unhandled shape fails loudly instead of silently. + +import ( + "strings" + "testing" +) + +func TestDuplicateIDWarning_NoCollision(t *testing.T) { + fns := map[string]FunctionInfo{} + if w := duplicateIDWarning(fns, "m.go:Stack.Len", FunctionInfo{FilePath: "m.go", StartLine: 10}); w != "" { + t.Errorf("expected no warning for a fresh id, got %q", w) + } + // A different id sharing the map must also not warn. + fns["m.go:Stack.Len"] = FunctionInfo{FilePath: "m.go", StartLine: 10} + if w := duplicateIDWarning(fns, "m.go:Queue.Len", FunctionInfo{FilePath: "m.go", StartLine: 20}); w != "" { + t.Errorf("expected no warning for a distinct id, got %q", w) + } +} + +func TestDuplicateIDWarning_Collision(t *testing.T) { + fns := map[string]FunctionInfo{ + "m.go:unknown.Len": {FilePath: "m.go", StartLine: 10}, + } + w := duplicateIDWarning(fns, "m.go:unknown.Len", FunctionInfo{FilePath: "m.go", StartLine: 20}) + if w == "" { + t.Fatal("expected a warning when funcID already present (silent overwrite), got none") + } + if !strings.Contains(w, "m.go:unknown.Len") { + t.Errorf("warning should name the colliding id; got %q", w) + } +} diff --git a/libs/openant-core/parsers/go/go_parser/extractor_generic_test.go b/libs/openant-core/parsers/go/go_parser/extractor_generic_test.go new file mode 100644 index 00000000..86cab48f --- /dev/null +++ b/libs/openant-core/parsers/go/go_parser/extractor_generic_test.go @@ -0,0 +1,142 @@ +package main + +// Regression tests for the generic-type key-collapse bug family in the Go parser. +// +// Bug 1 (extractor.go typeToString): a method on a generic type — `func (s *Stack[T]) Push()` — +// parses with an *ast.IndexExpr (one type param) or *ast.IndexListExpr (multiple) inside the +// receiver. typeToString had no case for either, so it fell to `default: return "unknown"`, +// making className "unknown". Every generic-type method across a repo then collapsed onto the +// fake class `unknown`, and DISTINCT types' same-named methods collided onto one unit id — +// silent data loss via map overwrite (output.Functions[id] = info). +// +// Bug 2 (callgraph.go analyzeCallExpr): generic CALL sites were dropped — `Gen[K,V]()` +// (*ast.IndexListExpr) had no case, and `obj.M[T]()` (*ast.IndexExpr whose .X is a +// *ast.SelectorExpr) was rejected by the `fun.X.(*ast.Ident)` guard — losing call edges. +// +// These tests exercise the stable public boundaries (Extract / extractCalls) so they compile +// against both pre-fix and post-fix sources; RED is demonstrated by running pre-fix. + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// writeTempGo writes src to /m.go and returns the path; the relPath from the repo root +// (dir) is therefore "m.go", so unit ids are "m.go:.". +func writeTempGo(t *testing.T, src string) (repo, file string) { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "m.go") + if err := os.WriteFile(path, []byte(src), 0o644); err != nil { + t.Fatalf("write temp go: %v", err) + } + return dir, path +} + +func extractIDs(t *testing.T, src string) map[string]FunctionInfo { + t.Helper() + repo, file := writeTempGo(t, src) + out, err := NewExtractor(repo).Extract([]string{file}) + if err != nil { + t.Fatalf("Extract: %v", err) + } + return out.Functions +} + +// Bug 1: generic receivers must key under the bare base type, never "unknown". +func TestGenericReceiver_ClassNameIsBaseType(t *testing.T) { + src := `package main + +type Stack[T any] struct{ items []T } +type Pair[K any, V any] struct{ k K; v V } +type Plain struct{} + +func (s *Stack[T]) Push(x T) {} // *ast.StarExpr -> *ast.IndexExpr +func (s Stack[T]) PeekVal() (z T) { return }// value receiver: *ast.IndexExpr directly +func (p *Pair[K, V]) Key() (k K) { return }// *ast.StarExpr -> *ast.IndexListExpr +func (p *Plain) Do() {} // control: *ast.Ident +` + fns := extractIDs(t, src) + want := []string{ + "m.go:Stack.Push", + "m.go:Stack.PeekVal", + "m.go:Pair.Key", + "m.go:Plain.Do", + } + for _, id := range want { + if _, ok := fns[id]; !ok { + t.Errorf("missing unit id %q; got ids %v", id, keys(fns)) + } + } + for id := range fns { + if strings.Contains(id, "unknown") { + t.Errorf("unit id %q contains the bogus class key 'unknown'", id) + } + } +} + +// Bug 1 (deeper, the data-loss regression): two DISTINCT generic types with a same-named method +// must produce two distinct units — pre-fix both collapsed to "m.go:unknown.Len" and the map +// overwrite silently dropped one. +func TestGenericTypes_NoUnitCollision(t *testing.T) { + src := `package main + +type Stack[T any] struct{} +type Queue[T any] struct{} + +func (s *Stack[T]) Len() int { return 0 } +func (q *Queue[T]) Len() int { return 1 } +` + fns := extractIDs(t, src) + if _, ok := fns["m.go:Stack.Len"]; !ok { + t.Errorf("missing m.go:Stack.Len; got %v", keys(fns)) + } + if _, ok := fns["m.go:Queue.Len"]; !ok { + t.Errorf("missing m.go:Queue.Len (silent collision/data-loss); got %v", keys(fns)) + } + if len(fns) != 2 { + t.Errorf("expected exactly 2 distinct method units, got %d: %v", len(fns), keys(fns)) + } +} + +// Bug 2: generic call sites must recover the called name (and receiver for generic methods). +func TestCallgraph_GenericCalls(t *testing.T) { + c := NewCallGraphBuilder("/repo") + fi := FunctionInfo{ + Name: "Run", + FilePath: "a/a.go", + Code: "func Run(){ " + + "Gen[int](); " + // *ast.IndexExpr{Ident} — already worked (control) + "Gen2[int,string](); " + // *ast.IndexListExpr — was dropped + "o.M[int](); " + // *ast.IndexExpr{SelectorExpr} — was dropped + "o.N[int,bool]() " + // *ast.IndexListExpr{SelectorExpr} — was dropped + "}", + } + calls := c.extractCalls(fi) + got := map[string]CallInfo{} + for _, ci := range calls { + got[ci.Name] = ci + } + for _, name := range []string{"Gen", "Gen2", "M", "N"} { + if _, ok := got[name]; !ok { + t.Errorf("generic call %q not recovered; got calls %+v", name, calls) + } + } + // generic METHOD calls must also recover the receiver + if ci, ok := got["M"]; ok && ci.Receiver != "o" { + t.Errorf("o.M[int](): expected receiver 'o', got %q", ci.Receiver) + } + if ci, ok := got["N"]; ok && ci.Receiver != "o" { + t.Errorf("o.N[int,bool](): expected receiver 'o', got %q", ci.Receiver) + } +} + +func keys(m map[string]FunctionInfo) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} diff --git a/libs/openant-core/parsers/go/go_parser/go_parser b/libs/openant-core/parsers/go/go_parser/go_parser deleted file mode 100755 index 198b846c..00000000 Binary files a/libs/openant-core/parsers/go/go_parser/go_parser and /dev/null differ diff --git a/libs/openant-core/parsers/go/test_pipeline.py b/libs/openant-core/parsers/go/test_pipeline.py index bba7551d..55330cde 100644 --- a/libs/openant-core/parsers/go/test_pipeline.py +++ b/libs/openant-core/parsers/go/test_pipeline.py @@ -52,6 +52,48 @@ from utilities.file_io import open_utf8, read_json, run_utf8, write_json +def normalize_go_function_records(raw_functions: dict) -> dict: + """Normalize Go FunctionInfo records to the snake_case consumer contract. + + The Go parser is a separate Go binary whose FunctionInfo records + (parsers/go/go_parser/types.go) use camelCase json keys + (``unitType``/``startLine``/``endLine``/``isExported``/``filePath``/ + ``className``), while every other parser emits snake_case. The Python + reachability/entry-point consumers (EntryPointDetector, etc.) read + snake_case — e.g. ``func_data.get('unit_type')``. Without normalization the + Go records' snake keys are ``None`` and any unit_type-based logic is + silently broken for Go (BUG-NEW 5). + + This maps the known FunctionInfo fields to snake_case, reading from either + shape so it is idempotent: already-snake records pass through unchanged. + Scope is the call_graph.json ``functions`` map only — the separate + analyzer_output.json camelCase contract is intentionally NOT touched. + """ + normalized: dict = {} + for func_id, fd in raw_functions.items(): + normalized[func_id] = { + 'name': fd.get('name', ''), + 'unit_type': fd.get('unit_type', fd.get('unitType', 'function')), + 'code': fd.get('code', ''), + 'file_path': fd.get('file_path', fd.get('filePath', '')), + 'start_line': fd.get('start_line', fd.get('startLine', 0)), + 'end_line': fd.get('end_line', fd.get('endLine', 0)), + 'package': fd.get('package', ''), + 'receiver': fd.get('receiver', ''), + 'is_exported': fd.get('is_exported', fd.get('isExported', False)), + 'class_name': fd.get('class_name', fd.get('className', '')), + 'decorators': fd.get('decorators', []), + # Schema-completeness (BUG-5 re-verify): carry the remaining + # FunctionInfo fields so a Go func record matches the snake_case + # shape the other parsers emit. No live consumer reads these today, + # but leaving them None is a latent drift for future consumers. + 'parameters': fd.get('parameters', []), + 'returns': fd.get('returns', []), + 'is_async': fd.get('is_async', fd.get('isAsync', False)), + } + return normalized + + def _stdout_supports_unicode() -> bool: """Return True if sys.stdout can emit the symbols we use for status. @@ -79,7 +121,7 @@ def _stdout_supports_unicode() -> bool: # Add parent directory to path for utilities import sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) from utilities.context_enhancer import ContextEnhancer -from utilities.agentic_enhancer import EntryPointDetector, ReachabilityAnalyzer +from utilities.agentic_enhancer import EntryPointDetector, ReachabilityAnalyzer, blackout_warning, library_seed_ids class ProcessingLevel(Enum): @@ -110,7 +152,8 @@ def __init__( processing_level: ProcessingLevel = ProcessingLevel.ALL, skip_tests: bool = False, depth: int = 3, - name: str = None + name: str = None, + library_mode: bool = False ): self.repo_path = os.path.abspath(repo_path) self.output_dir = output_dir or os.path.join(os.path.dirname(__file__), 'test_output') @@ -121,6 +164,7 @@ def __init__( self.skip_tests = skip_tests self.depth = depth self.dataset_name = name + self.library_mode = library_mode # Go parser binary location self.go_parser = os.path.join(self.parser_dir, 'go_parser', 'go_parser') @@ -305,21 +349,10 @@ def run_go_parser_all(self) -> bool: dataset_for_cg = read_json(self.dataset_file) raw_functions = analyzer.get("functions", {}) - # Normalise to the camelCase shape EntryPointDetector expects. - normalized_functions = { - func_id: { - 'name': fd.get('name', ''), - 'unitType': fd.get('unit_type', fd.get('unitType', 'function')), - 'code': fd.get('code', ''), - 'filePath': fd.get('file_path', fd.get('filePath', '')), - 'startLine': fd.get('start_line', fd.get('startLine', 0)), - 'endLine': fd.get('end_line', fd.get('endLine', 0)), - 'package': fd.get('package', ''), - 'receiver': fd.get('receiver', ''), - 'isExported': fd.get('is_exported', fd.get('isExported', False)), - } - for func_id, fd in raw_functions.items() - } + # Normalise to the snake_case shape the Python consumers read + # (EntryPointDetector reads func_data.get('unit_type'), etc.). + # Idempotent: already-snake records pass through unchanged. + normalized_functions = normalize_go_function_records(raw_functions) call_graph: dict = {} reverse_call_graph: dict = {} @@ -375,21 +408,11 @@ def apply_reachability_filter(self) -> bool: functions = analyzer.get("functions", {}) - # Convert to expected format for EntryPointDetector - # Go parser uses snake_case, EntryPointDetector expects camelCase - normalized_functions = {} - for func_id, func_data in functions.items(): - normalized_functions[func_id] = { - 'name': func_data.get('name', ''), - 'unitType': func_data.get('unit_type', func_data.get('unitType', 'function')), - 'code': func_data.get('code', ''), - 'filePath': func_data.get('file_path', func_data.get('filePath', '')), - 'startLine': func_data.get('start_line', func_data.get('startLine', 0)), - 'endLine': func_data.get('end_line', func_data.get('endLine', 0)), - 'package': func_data.get('package', ''), - 'receiver': func_data.get('receiver', ''), - 'isExported': func_data.get('is_exported', func_data.get('isExported', False)), - } + # Convert to the snake_case shape the Python consumers read. + # The Go binary's analyzer_output uses camelCase FunctionInfo keys; + # EntryPointDetector / ReachabilityAnalyzer read snake_case + # (func_data.get('unit_type'), etc.). Idempotent normalization. + normalized_functions = normalize_go_function_records(functions) # Load call graph from dataset (go_parser puts it in statistics) dataset = read_json(self.dataset_file) @@ -412,6 +435,11 @@ def apply_reachability_filter(self) -> bool: detector = EntryPointDetector(normalized_functions, call_graph) self.entry_points = detector.detect_entry_points() + # Library-mode: seed the exported public API (a library has no + # main/route/CLI marker). Union-only — never drops a real entry point. + if self.library_mode: + self.entry_points = self.entry_points | library_seed_ids(normalized_functions) + # Build reachability analyzer reachability = ReachabilityAnalyzer( functions=normalized_functions, @@ -445,6 +473,13 @@ def apply_reachability_filter(self) -> bool: "reduction_percentage": round((1 - len(filtered_units) / original_count) * 100, 1) if original_count > 0 else 0 } + _blackout = blackout_warning(detector.entry_point_details, original_count, + len(filtered_units), + library_mode=getattr(self, "library_mode", False)) + if _blackout: + dataset["metadata"]["reachability_filter"]["warning"] = _blackout + print(f" [Warning] {_blackout}", file=sys.stderr) + # Write filtered dataset write_json(self.dataset_file, dataset) @@ -1183,6 +1218,11 @@ def main(): default=None, help='Dataset name (default: derived from repo path)' ) + parser.add_argument( + '--library-mode', + action='store_true', + help='Seed the exported public API as entry points (for libraries with no main/route/CLI)' + ) args = parser.parse_args() @@ -1205,7 +1245,8 @@ def main(): processing_level=processing_level, skip_tests=args.skip_tests, depth=args.depth, - name=args.name + name=args.name, + library_mode=args.library_mode ) results = pipeline.run_full_pipeline() diff --git a/libs/openant-core/parsers/javascript/test_pipeline.py b/libs/openant-core/parsers/javascript/test_pipeline.py index 6cf8911d..6a32d594 100644 --- a/libs/openant-core/parsers/javascript/test_pipeline.py +++ b/libs/openant-core/parsers/javascript/test_pipeline.py @@ -78,7 +78,7 @@ def _stdout_supports_unicode() -> bool: # Add parent directory to path for utilities import sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) from utilities.context_enhancer import ContextEnhancer -from utilities.agentic_enhancer import EntryPointDetector, ReachabilityAnalyzer +from utilities.agentic_enhancer import EntryPointDetector, ReachabilityAnalyzer, blackout_warning, library_seed_ids class ProcessingLevel(Enum): @@ -103,7 +103,8 @@ def __init__( processing_level: ProcessingLevel = ProcessingLevel.ALL, skip_tests: bool = False, depth: int = 3, - name: str = None + name: str = None, + library_mode: bool = False ): self.repo_path = os.path.abspath(repo_path) self.output_dir = output_dir or os.path.join(os.path.dirname(__file__), 'test_output') @@ -114,6 +115,7 @@ def __init__( self.skip_tests = skip_tests self.depth = depth self.dataset_name = name + self.library_mode = library_mode # Component locations # repository_scanner.js and unit_generator.js are in this package @@ -623,6 +625,11 @@ def apply_reachability_filter(self) -> bool: detector = EntryPointDetector(functions, call_graph) self.entry_points = detector.detect_entry_points() + # Library-mode: seed the exported public API (a library has no + # main/route/CLI marker). Union-only — never drops a real entry point. + if self.library_mode: + self.entry_points = self.entry_points | library_seed_ids(functions) + # Build reachability analyzer reachability = ReachabilityAnalyzer( functions=functions, @@ -659,6 +666,13 @@ def apply_reachability_filter(self) -> bool: "reduction_percentage": round((1 - len(filtered_units) / original_count) * 100, 1) if original_count > 0 else 0 } + _blackout = blackout_warning(detector.entry_point_details, original_count, + len(filtered_units), + library_mode=getattr(self, "library_mode", False)) + if _blackout: + dataset["metadata"]["reachability_filter"]["warning"] = _blackout + print(f" [Warning] {_blackout}", file=sys.stderr) + # Write filtered dataset write_json(self.dataset_file, dataset) @@ -1318,6 +1332,7 @@ def main(): skip_tests = False depth = 3 name = None + library_mode = False else: parser = argparse.ArgumentParser( description='Test the parser pipeline on a repository', @@ -1389,6 +1404,11 @@ def main(): default=None, help='Dataset name (default: derived from repo path)' ) + parser.add_argument( + '--library-mode', + action='store_true', + help='Seed the exported public API as entry points (for libraries with no main/route/CLI)' + ) args = parser.parse_args() repo_path = args.repo_path @@ -1400,6 +1420,7 @@ def main(): skip_tests = args.skip_tests depth = args.depth name = args.name + library_mode = args.library_mode if not os.path.exists(repo_path): print(f"Error: Repository not found: {repo_path}") @@ -1423,7 +1444,8 @@ def main(): processing_level=processing_level, skip_tests=skip_tests, depth=depth, - name=name + name=name, + library_mode=library_mode ) results = pipeline.run_full_pipeline() diff --git a/libs/openant-core/parsers/javascript/typescript_analyzer.js b/libs/openant-core/parsers/javascript/typescript_analyzer.js index baf219c8..30336f46 100644 --- a/libs/openant-core/parsers/javascript/typescript_analyzer.js +++ b/libs/openant-core/parsers/javascript/typescript_analyzer.js @@ -219,6 +219,68 @@ class TypeScriptAnalyzer { /** * Extract all functions/methods from a source file */ + // Module-level FunctionDeclarations: top-level OR nested only inside blocks / + // control-flow statements (if/else, try/catch, for/while, switch, bare {}) — + // NOT inside another function/method/accessor (their text already rides inside + // the parent unit) and NOT inside a class `static {}` block (block-scoped to + // the initializer, callable nowhere). `getFunctions()` returned only top-level + // declarations, so block-scoped functions were silently dropped from both the + // inventory and the call graph. Returns [{node, id}] with COLLISION-ONLY + // `#L` disambiguation (sibling-block same-name functions are both + // runtime-reachable, so keep both; unique names keep the plain `path:name` + // id). Both the inventory and call-graph builders iterate this same list so + // their keys match exactly (the `len(callGraph) === len(functions)` lockstep). + // The named FunctionDeclaration nodes that are module-level (top-level OR only + // block-nested), excluding function-nested and static-block-scoped ones. Used + // by the inventory builder, the call-graph builder, AND the resolver — all + // three must see the same set so a block function is a unit, has its own + // edges, and is resolvable as a call target. + _moduleLevelFunctionNodes(sourceFile) { + const STOP = new Set([ + ts.SyntaxKind.FunctionDeclaration, + ts.SyntaxKind.FunctionExpression, + ts.SyntaxKind.ArrowFunction, + ts.SyntaxKind.MethodDeclaration, + ts.SyntaxKind.Constructor, + ts.SyntaxKind.GetAccessor, + ts.SyntaxKind.SetAccessor, + ts.SyntaxKind.ClassStaticBlockDeclaration, + ]); + const isModuleLevel = (fn) => { + for (let p = fn.getParent(); p; p = p.getParent()) { + if (STOP.has(p.getKind())) return false; + } + return true; + }; + return sourceFile + .getDescendantsOfKind(ts.SyntaxKind.FunctionDeclaration) + .filter((fn) => fn.getName() && isModuleLevel(fn)); + } + + _moduleLevelFunctionEntries(sourceFile, relativePath) { + const fns = this._moduleLevelFunctionNodes(sourceFile); + const countByName = {}; + for (const fn of fns) { + const n = fn.getName(); + countByName[n] = (countByName[n] || 0) + 1; + } + const seen = new Set(); + return fns.map((fn) => { + const name = fn.getName(); + let id = `${relativePath}:${name}`; + if (countByName[name] > 1) { + // collision-only: keep both, deterministic by source line (never by + // emit order). Colon-free `#L` suffix survives the downstream + // `split(':')[0]` (file) / `rsplit(':',1)[-1]` (name) id contracts. + let uid = `${id}#L${fn.getStartLineNumber()}`; + while (seen.has(uid)) uid = `${uid}.x`; + seen.add(uid); + id = uid; + } + return { node: fn, id }; + }); + } + extractFunctionsFromFile(sourceFile) { // Always emit POSIX-style relative paths so functionId values are // stable across platforms (Python downstream consumers and dataset @@ -227,13 +289,13 @@ class TypeScriptAnalyzer { path.relative(this.repoPath, sourceFile.getFilePath()), ); - // Extract function declarations - for (const func of sourceFile.getFunctions()) { + // Extract function declarations (top-level + module-level block-scoped). + for (const { node: func, id: functionId } of this._moduleLevelFunctionEntries( + sourceFile, + relativePath, + )) { const name = func.getName(); - if (!name) continue; - const code = func.getFullText(); - const functionId = `${relativePath}:${name}`; this.functions[functionId] = { name: name, code: code, @@ -1249,12 +1311,13 @@ class TypeScriptAnalyzer { path.relative(this.repoPath, sourceFile.getFilePath()), ); - // Analyze function declarations - for (const func of sourceFile.getFunctions()) { - const name = func.getName(); - if (!name) continue; - - const callerId = `${relativePath}:${name}`; + // Analyze function declarations (same enumeration + id scheme as the + // inventory builder, so callGraph keys match functions keys exactly — + // a block function must get its REAL outgoing edges, not a backstop []). + for (const { node: func, id: callerId } of this._moduleLevelFunctionEntries( + sourceFile, + relativePath, + )) { this.callGraph[callerId] = this.extractCallsFromFunction( func, relativePath, @@ -1529,9 +1592,10 @@ function extractSingleFunction(filePath, functionRef) { } } - // 2. Try standalone function declarations + // 2. Try standalone function declarations (incl. module-level block-scoped, + // so a call TO a function defined inside an if/try/for block resolves). if (!foundFunction) { - for (const func of sourceFile.getFunctions()) { + for (const func of this._moduleLevelFunctionNodes(sourceFile)) { if (func.getName() === functionName) { foundFunction = { node: func, diff --git a/libs/openant-core/parsers/php/call_graph_builder.py b/libs/openant-core/parsers/php/call_graph_builder.py index 43546313..566e79c1 100644 --- a/libs/openant-core/parsers/php/call_graph_builder.py +++ b/libs/openant-core/parsers/php/call_graph_builder.py @@ -134,6 +134,8 @@ def __init__(self, extractor_output: Dict, options: Optional[Dict] = None): self.functions_by_name: Dict[str, List[str]] = {} self.functions_by_file: Dict[str, List[str]] = {} self.methods_by_class: Dict[str, List[str]] = {} + # class_key -> list of trait names the class composes via in-class `use`. + self.traits_by_class: Dict[str, List[str]] = {} self._build_indexes() @@ -162,6 +164,13 @@ def _build_indexes(self) -> None: self.methods_by_class[class_key] = [] self.methods_by_class[class_key].append(func_id) + # Index each class's composed traits (in-class `use TraitName;`) so a + # $this->/self:: call can fall back to a method pulled in from a trait. + for class_key, class_data in self.classes.items(): + traits = class_data.get('traits') + if traits: + self.traits_by_class[class_key] = list(traits) + def _is_builtin(self, name: str) -> bool: """Check if name is a PHP builtin or common function.""" return name.lower() in PHP_BUILTINS # PHP function names are case-insensitive @@ -174,19 +183,27 @@ def _extract_calls_from_code(self, code: str, caller_id: str) -> Set[str]: caller_class = caller_func.get('class_name') caller_namespace = caller_func.get('namespace_name') + # The extractor stores each function/method body as a raw PHP fragment + # WITHOUT a leading " Set[str]: def _resolve_call_node(self, node, source: bytes, caller_file: str, caller_class: Optional[str], - caller_namespace: Optional[str] = None) -> Optional[str]: + caller_namespace: Optional[str] = None, + root=None) -> Optional[str]: """Resolve a tree-sitter call node to a function ID.""" if node.type == 'function_call_expression': return self._resolve_function_call(node, source, caller_file, caller_class, - caller_namespace) + caller_namespace, root) elif node.type == 'member_call_expression': return self._resolve_member_call(node, source, caller_file, caller_class) elif node.type == 'scoped_call_expression': @@ -210,7 +228,8 @@ def _resolve_call_node(self, node, source: bytes, caller_file: str, def _resolve_function_call(self, node, source: bytes, caller_file: str, caller_class: Optional[str], - caller_namespace: Optional[str] = None) -> Optional[str]: + caller_namespace: Optional[str] = None, + root=None) -> Optional[str]: """Resolve a simple function call like func().""" func_name = None @@ -224,6 +243,12 @@ def _resolve_function_call(self, node, source: bytes, caller_file: str, if '\\' in func_name: func_name = func_name.rsplit('\\', 1)[-1] break + elif child.type == 'variable_name': + # Variable-function call like $f(). Follow a single + # string-literal binding ($f = 'helper';) to recover the name. + var_name = source[child.start_byte:child.end_byte].decode('utf-8', errors='replace') + func_name = self._resolve_variable_function(var_name, root, source) + break if not func_name: return None @@ -292,6 +317,56 @@ def _resolve_new(self, node, source: bytes, caller_file: str, return None return self._resolve_class_call(class_name, '__construct', caller_file) + def _resolve_variable_function(self, var_name: str, root, + source: bytes) -> Optional[str]: + """Follow a single string-literal binding for a $var() callee. + + Scans the enclosing function body for assignments to ``var_name``. + Only a single, unambiguous string-literal binding + (``$f = 'helper';``) is followed; if the variable is assigned more + than once, or from a non-literal, resolution is declined for + precision (no guessing). + """ + if root is None: + return None + literal_names: Set[str] = set() + non_literal = False + + stack = [root] + while stack: + n = stack.pop() + if n.type == 'assignment_expression': + children = [c for c in n.children if c.type not in ('=',)] + # Shape: = + if len(children) >= 2 and children[0].type == 'variable_name': + lhs = source[children[0].start_byte:children[0].end_byte].decode( + 'utf-8', errors='replace') + if lhs == var_name: + rhs = children[1] + literal = self._string_literal_value(rhs, source) + if literal is not None: + literal_names.add(literal) + else: + non_literal = True + stack.extend(n.children) + + # Single unambiguous string binding only. + if non_literal or len(literal_names) != 1: + return None + return next(iter(literal_names)) + + @staticmethod + def _string_literal_value(node, source: bytes) -> Optional[str]: + """Return the content of a string-literal node, else None.""" + if node.type != 'string': + return None + for child in node.children: + if child.type == 'string_content': + return source[child.start_byte:child.end_byte].decode( + 'utf-8', errors='replace') + # Empty string literal ('') has no string_content child. + return '' + def _resolve_member_call(self, node, source: bytes, caller_file: str, caller_class: Optional[str]) -> Optional[str]: """Resolve a member call like $obj->method().""" @@ -348,27 +423,18 @@ def _resolve_scoped_call(self, node, source: bytes, caller_file: str, if scope in ('self', 'static') and caller_class: return self._resolve_self_call(method_name, caller_file, caller_class) - # parent::method() - resolve in the superclass (extends), not the caller's own class. + # parent::method() - the method is inherited from the parent class, + # which may be defined in a different file. Resolve via the + # class->parent index, then a cross-file class-method lookup. if scope == 'parent' and caller_class: - superclass = self._superclass_of(caller_file, caller_class) - if not superclass: - return None - if '\\' in superclass: - superclass = superclass.rsplit('\\', 1)[-1] # `extends App\Base` -> match `Base` - # Resolve in the superclass only; do NOT fall back to the caller's own class, which would - # mis-link an overriding child's parent:: call to the child's own method. - return self._resolve_class_call(superclass, method_name, caller_file) + parent_class = self._resolve_parent_class(caller_file, caller_class) + if parent_class: + return self._resolve_class_call(parent_class, method_name, caller_file) + return None # ClassName::method() return self._resolve_class_call(scope, method_name, caller_file) - def _superclass_of(self, caller_file: str, caller_class: str) -> Optional[str]: - """Return the superclass (extends) name of caller_class defined in caller_file, or None.""" - for class_data in self.classes.values(): - if class_data.get('name') == caller_class and class_data.get('file_path') == caller_file: - return class_data.get('superclass') - return None - def _resolve_simple_call(self, func_name: str, caller_file: str, caller_class: Optional[str], caller_namespace: Optional[str] = None) -> Optional[str]: @@ -438,6 +504,15 @@ def _resolve_self_call(self, method_name: str, caller_file: str, if func_data.get('name') == method_name: return func_id + # Fall back to methods composed in via traits (`use TraitName;`). A trait + # method is invoked exactly like an own method ($this->m()/self::m()), but + # it lives under the trait's own class_key, so resolve it there. The trait + # may be declared in a different file, hence the cross-file lookup. + for trait_name in self.traits_by_class.get(class_key, []): + resolved = self._resolve_class_call(trait_name, method_name, caller_file) + if resolved: + return resolved + return None def _resolve_class_call(self, class_name: str, method_name: str, @@ -461,6 +536,33 @@ def _resolve_class_call(self, class_name: str, method_name: str, return None + def _resolve_parent_class(self, caller_file: str, + caller_class: str) -> Optional[str]: + """Return the parent (superclass) name of caller_class, if known. + + The class index records each class's ``superclass`` (the ``extends`` + target). The parent class may be defined in a different file, so a + same-file lookup is tried first, then any file declaring the class. + """ + # Same-file class declaration first (most precise). + class_data = self.classes.get(f"{caller_file}:{caller_class}") + if class_data and class_data.get('superclass'): + return self._strip_namespace(class_data['superclass']) + + # Fall back to any class with this name across files. + for key, data in self.classes.items(): + if key.endswith(f":{caller_class}") and data.get('superclass'): + return self._strip_namespace(data['superclass']) + + return None + + @staticmethod + def _strip_namespace(name: str) -> str: + """Reduce a possibly namespace-qualified class name to its last segment.""" + if '\\' in name: + return name.rsplit('\\', 1)[-1] + return name + def _extract_calls_regex(self, code: str, caller_id: str) -> Set[str]: """Fallback regex-based call extraction for unparseable code.""" calls = set() diff --git a/libs/openant-core/parsers/php/function_extractor.py b/libs/openant-core/parsers/php/function_extractor.py index 582a6892..d68b1b72 100644 --- a/libs/openant-core/parsers/php/function_extractor.py +++ b/libs/openant-core/parsers/php/function_extractor.py @@ -138,6 +138,23 @@ def _is_static_method(self, node, source: bytes) -> bool: return True return False + def _extract_trait_names(self, use_node, source: bytes) -> List[str]: + """Extract trait names from an in-class `use_declaration` node. + + Handles grouped uses (`use A, B\\C;`). Each trait is a `name` or + `qualified_name` child; namespace-qualified names are reduced to their + last segment so resolution matches the trait's unqualified class name. + """ + names = [] + for child in use_node.children: + if child.type in ('name', 'qualified_name'): + trait = self._node_text(child, source) + if '\\' in trait: + trait = trait.rsplit('\\', 1)[-1] + if trait: + names.append(trait) + return names + def _get_visibility(self, node, source: bytes) -> Optional[str]: """Extract visibility modifier from a method_declaration node.""" for child in node.children: @@ -306,6 +323,7 @@ def _extract_functions_from_tree(self, tree, source: bytes, file_path: Path, if new_class_name: class_id = f"{relative_path}:{new_class_name}" methods = [] + traits = [] # Find declaration_list (class body) body_node = node.child_by_field_name('body') if body_node is None: @@ -323,6 +341,13 @@ def _extract_functions_from_tree(self, tree, source: bytes, file_path: Path, methods.append(f"static:{mname}") else: methods.append(mname) + elif child.type == 'use_declaration': + # In-class `use TraitA, NS\TraitB;` composes traits into the class. + # tree-sitter-php emits this as `use_declaration` (distinct from the + # top-level `namespace_use_declaration`), with each trait as a + # `name`/`qualified_name` child. Record the unqualified trait name so + # the call-graph builder can resolve $this->/self:: into the trait. + traits.extend(self._extract_trait_names(child, source)) self.classes[class_id] = { 'name': new_class_name, @@ -330,6 +355,7 @@ def _extract_functions_from_tree(self, tree, source: bytes, file_path: Path, 'start_line': node.start_point[0] + 1, 'end_line': node.end_point[0] + 1, 'methods': methods, + 'traits': traits, 'superclass': superclass, 'interfaces': interfaces, 'namespace_name': namespace_name, @@ -420,6 +446,50 @@ def _extract_functions_from_tree(self, tree, source: bytes, file_path: Path, stack.append((child, new_trait_name, namespace_name)) continue + elif node.type == 'enum_declaration': + # PHP 8.1+ enums are class-like callable containers; register + # the enum as a class so its methods get qualified ids + # (Enum.method) and class context, mirroring trait/interface. + name_node = node.child_by_field_name('name') + new_enum_name = self._node_text(name_node, source) if name_node else None + + if new_enum_name: + class_id = f"{relative_path}:{new_enum_name}" + methods = [] + body_node = node.child_by_field_name('body') + if body_node is None: + for child in node.children: + if child.type == 'enum_declaration_list': + body_node = child + break + + if body_node: + for child in body_node.children: + if child.type == 'method_declaration': + mname = self._get_function_name(child, source) + if mname: + if self._is_static_method(child, source): + methods.append(f"static:{mname}") + else: + methods.append(mname) + + self.classes[class_id] = { + 'name': new_enum_name, + 'file_path': relative_path, + 'start_line': node.start_point[0] + 1, + 'end_line': node.end_point[0] + 1, + 'methods': methods, + 'superclass': None, + 'interfaces': [], + 'namespace_name': namespace_name, + } + self.stats['total_classes'] += 1 + + if body_node: + for child in reversed(body_node.children): + stack.append((child, new_enum_name, namespace_name)) + continue + elif node.type == 'namespace_definition': # Extract namespace name name_node = node.child_by_field_name('name') @@ -428,18 +498,84 @@ def _extract_functions_from_tree(self, tree, source: bytes, file_path: Path, # Recurse into namespace body body_node = node.child_by_field_name('body') if body_node is None: - # Namespace without braces covers rest of file; recurse children directly - for child in reversed(node.children): - if child.type not in ('namespace', 'name', ';'): - stack.append((child, class_name, new_namespace_name)) + # Braceless 'namespace App\Svc;': the declarations it covers + # are SIBLINGS of this node, not children -- they are pushed + # by the container's traversal (see _push_children, which + # carries the braceless namespace forward to siblings). The + # node's own children (namespace/name/;) have nothing to + # extract, so there is nothing to recurse here. + pass else: for child in reversed(body_node.children): stack.append((child, class_name, new_namespace_name)) continue # Don't walk children again + elif node.type == 'anonymous_class': + # `new class { ... }` (PHP 7+) has no source name. Without a synthetic + # identity its methods fall through the catch-all else with the OUTER + # class_name (None at top level), so they're keyed as bare functions and + # two distinct anonymous classes that both define e.g. handle() collide on + # one id (the later silently overwrites the earlier). Synthesize a stable, + # location-based name so each anonymous class is distinct and its methods + # are qualified (class@anonymous::.method). Line AND column are + # both needed: two `new class {}` on one physical line share a start line, + # so column is what keeps them distinct (else they'd still collide). + anon_name = ( + f"class@anonymous:{node.start_point[0] + 1}:{node.start_point[1]}" + ) + body_node = None + for child in node.children: + if child.type == 'declaration_list': + body_node = child + break + + if body_node: + methods = [] + for child in body_node.children: + if child.type == 'method_declaration': + mname = self._get_function_name(child, source) + if mname: + if self._is_static_method(child, source): + methods.append(f"static:{mname}") + else: + methods.append(mname) + + self.classes[f"{relative_path}:{anon_name}"] = { + 'name': anon_name, + 'file_path': relative_path, + 'start_line': node.start_point[0] + 1, + 'end_line': node.end_point[0] + 1, + 'methods': methods, + 'superclass': None, + 'interfaces': [], + 'namespace_name': namespace_name, + } + self.stats['total_classes'] += 1 + + for child in reversed(body_node.children): + stack.append((child, anon_name, namespace_name)) + continue # Don't walk children again + else: - for child in reversed(node.children): - stack.append((child, class_name, namespace_name)) + self._push_children(stack, node.children, source, class_name, namespace_name) + + def _push_children(self, stack, children, source: bytes, + class_name: Optional[str], namespace_name: Optional[str]) -> None: + """Push a node's children onto the traversal stack, honoring a braceless + ``namespace App\\X;`` declaration: such a node has no body, and the + declarations it governs are its FOLLOWING SIBLINGS (until the next + namespace declaration). Propagate that namespace to those siblings.""" + current_ns = namespace_name + ns_overrides = {} + for child in children: + if child.type == 'namespace_definition' and child.child_by_field_name('body') is None: + name_node = child.child_by_field_name('name') + if name_node is not None: + current_ns = self._node_text(name_node, source) + continue + ns_overrides[id(child)] = current_ns + for child in reversed(children): + stack.append((child, class_name, ns_overrides.get(id(child), namespace_name))) def _process_function_node(self, node, source: bytes, relative_path: str, class_name: Optional[str], namespace_name: Optional[str], @@ -450,7 +586,16 @@ def _process_function_node(self, node, source: bytes, relative_path: str, return code = self._node_text(node, source) - start_line = node.start_point[0] + 1 # tree-sitter is 0-indexed + # tree-sitter is 0-indexed. The method_declaration node spans any + # leading PHP8 attribute_list (e.g. #[Route(...)]), so node.start_point + # would point at the attribute line. Anchor start_line at the actual + # declaration (first non-attribute child) instead. + decl_node = node + for child in node.children: + if child.type != 'attribute_list': + decl_node = child + break + start_line = decl_node.start_point[0] + 1 end_line = node.end_point[0] + 1 parameters = self._get_parameters(node, source) @@ -482,7 +627,7 @@ def _process_function_node(self, node, source: bytes, relative_path: str, 'unit_type': unit_type, } - self.functions[func_id] = func_data + self._store_function(func_id, func_data) self.stats['total_functions'] += 1 if class_name: @@ -524,7 +669,7 @@ def _process_closure_node(self, node, source: bytes, relative_path: str, func_id = f"{relative_path}:{qualified_name}" - self.functions[func_id] = { + self._store_function(func_id, { 'name': name, 'qualified_name': qualified_name, 'file_path': relative_path, @@ -536,11 +681,35 @@ def _process_closure_node(self, node, source: bytes, relative_path: str, 'parameters': parameters, 'is_static': False, 'unit_type': 'closure', - } + }) self.stats['total_functions'] += 1 self.stats['standalone_functions'] += 1 self.stats['by_type']['closure'] = self.stats['by_type'].get('closure', 0) + 1 + def _store_function(self, func_id: str, func_data: dict) -> str: + """Insert a function unit, keeping BOTH on a same-(file,name) collision. + + PHP forbids plain redefinition (fatal), so legal duplicates arise only + from mutually-exclusive conditional branches — an `if/else` or a + defensive double `if(!function_exists('x')){...}` — where which branch + is live is environment-dependent and the EARLIER branch may be the one + that runs. Keying solely on `qualified_name` let the later branch + overwrite the earlier (a silent false negative). Disambiguate + DETERMINISTICALLY by source line (`#L`); the earlier-in-source + unit keeps the clean id. Mirrors the Python extractor's `_store_function`. + """ + if func_id not in self.functions: + self.functions[func_id] = func_data + return func_id + line = func_data.get('start_line', 0) + unique_id = f"{func_id}#L{line}" + n = 2 + while unique_id in self.functions: + unique_id = f"{func_id}#L{line}.{n}" + n += 1 + self.functions[unique_id] = func_data + return unique_id + def _extract_module_level_unit(self, tree, source: bytes, relative_path: str) -> None: """Synthesise a `module_level` unit for top-level procedural statements. @@ -625,7 +794,7 @@ def process_file(self, file_path: Path) -> None: self.stats['files_with_errors'] += 1 return - relative_path = str(file_path.relative_to(self.repo_path)) + relative_path = file_path.relative_to(self.repo_path).as_posix() try: tree = self.parser.parse(source) diff --git a/libs/openant-core/parsers/php/test_pipeline.py b/libs/openant-core/parsers/php/test_pipeline.py index 9947bdd5..e3f9960a 100644 --- a/libs/openant-core/parsers/php/test_pipeline.py +++ b/libs/openant-core/parsers/php/test_pipeline.py @@ -48,7 +48,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) from utilities.file_io import open_utf8, read_json, run_utf8, write_json from utilities.context_enhancer import ContextEnhancer -from utilities.agentic_enhancer import EntryPointDetector, ReachabilityAnalyzer +from utilities.agentic_enhancer import EntryPointDetector, ReachabilityAnalyzer, blackout_warning, library_seed_ids # Local imports from repository_scanner import RepositoryScanner @@ -78,7 +78,8 @@ def __init__( processing_level: ProcessingLevel = ProcessingLevel.ALL, skip_tests: bool = False, depth: int = 3, - name: str = None + name: str = None, + library_mode: bool = False ): self.repo_path = os.path.abspath(repo_path) self.output_dir = output_dir or os.path.join(os.path.dirname(__file__), 'test_output') @@ -89,6 +90,7 @@ def __init__( self.skip_tests = skip_tests self.depth = depth self.dataset_name = name + self.library_mode = library_mode # Pipeline artifacts self.scan_results_file = None @@ -283,6 +285,11 @@ def apply_reachability_filter(self) -> bool: detector = EntryPointDetector(normalized_functions, call_graph) self.entry_points = detector.detect_entry_points() + # Library-mode: seed the exported public API (a library has no + # main/route/CLI marker). Union-only — never drops a real entry point. + if self.library_mode: + self.entry_points = self.entry_points | library_seed_ids(normalized_functions) + # Build reachability reachability = ReachabilityAnalyzer( functions=normalized_functions, @@ -314,6 +321,13 @@ def apply_reachability_filter(self) -> bool: "reduction_percentage": round((1 - len(filtered_units) / original_count) * 100, 1) if original_count > 0 else 0 } + _blackout = blackout_warning(detector.entry_point_details, original_count, + len(filtered_units), + library_mode=getattr(self, "library_mode", False)) + if _blackout: + dataset["metadata"]["reachability_filter"]["warning"] = _blackout + print(f" [Warning] {_blackout}", file=sys.stderr) + write_json(self.dataset_file, dataset) elapsed = (datetime.now() - start_time).total_seconds() @@ -1003,6 +1017,11 @@ def main(): default=None, help='Dataset name (default: derived from repo path)' ) + parser.add_argument( + '--library-mode', + action='store_true', + help='Seed the exported public API as entry points (for libraries with no main/route/CLI)' + ) args = parser.parse_args() @@ -1024,7 +1043,8 @@ def main(): processing_level=processing_level, skip_tests=args.skip_tests, depth=args.depth, - name=args.name + name=args.name, + library_mode=args.library_mode ) results = pipeline.run_full_pipeline() diff --git a/libs/openant-core/parsers/php/unit_generator.py b/libs/openant-core/parsers/php/unit_generator.py index 63f9fffa..c787f71b 100644 --- a/libs/openant-core/parsers/php/unit_generator.py +++ b/libs/openant-core/parsers/php/unit_generator.py @@ -160,7 +160,11 @@ def create_unit(self, func_id: str, func_data: Dict) -> Dict: file_path = func_data.get('file_path', '') func_name = func_data.get('name', '') class_name = func_data.get('class_name') - namespace = func_data.get('namespace') + # The extractor (function_extractor.py) writes the declared namespace + # under 'namespace_name'; read that canonical key so it reaches the + # unit instead of always being None (key-drift bug). + namespace = func_data.get('namespace_name') + is_static = func_data.get('is_static', False) unit_type = func_data.get('unit_type', 'function') # Get upstream dependencies (functions this calls) @@ -239,6 +243,7 @@ def create_unit(self, func_id: str, func_data: Dict) -> Dict: 'metadata': { 'visibility': func_data.get('visibility', 'public'), 'namespace': namespace, + 'is_static': is_static, 'parameters': func_data.get('parameters', []), 'generator': 'php_unit_generator.py', 'direct_calls': direct_calls, @@ -307,7 +312,8 @@ def generate_analyzer_output(self) -> Dict: 'endLine': func_data.get('end_line', 0), 'visibility': func_data.get('visibility', 'public'), 'isExported': True, # PHP doesn't have explicit exports - 'namespace': func_data.get('namespace'), + 'namespace': func_data.get('namespace_name'), + 'isStatic': func_data.get('is_static', False), 'parameters': func_data.get('parameters', []), 'className': func_data.get('class_name'), } diff --git a/libs/openant-core/parsers/python/ast_parser.py b/libs/openant-core/parsers/python/ast_parser.py index 7f9b7c86..6413b7ab 100644 --- a/libs/openant-core/parsers/python/ast_parser.py +++ b/libs/openant-core/parsers/python/ast_parser.py @@ -127,7 +127,7 @@ def _create_unit(self, method: str, path: str, handler: str, "code": { "primary_code": code, "primary_origin": { - "file_path": str(file_path.relative_to(self.repo_path)), + "file_path": file_path.relative_to(self.repo_path).as_posix(), "start_line": start_line, "end_line": end_line, "function_name": handler, @@ -153,7 +153,7 @@ def _create_unit(self, method: str, path: str, handler: str, "metadata": { "parser": "python_ast_parser.py", "framework": self.framework, - "route_file": str(file_path.relative_to(self.repo_path)), + "route_file": file_path.relative_to(self.repo_path).as_posix(), "route_line": start_line } } diff --git a/libs/openant-core/parsers/python/call_graph_builder.py b/libs/openant-core/parsers/python/call_graph_builder.py index 9cd73984..4a1d38ed 100644 --- a/libs/openant-core/parsers/python/call_graph_builder.py +++ b/libs/openant-core/parsers/python/call_graph_builder.py @@ -186,10 +186,18 @@ def _extract_calls_from_code(self, code: str, caller_id: str) -> Set[str]: # Local variable -> constructor-type map, so that `v = ClassName(); v.method()` # dispatches to the bound type's method. local_types = self._collect_local_types(tree) + # Receiver names that refer to the current instance/class: self/cls plus any + # local name single-bound to them (obj = self; obj.method()). + self_aliases = {'self', 'cls'} | self._collect_self_aliases(tree) + + # Local function-value aliases within this caller: `fn = helper` makes a + # subsequent `fn()` a call to `helper`. Resolve the alias target to a + # function ID up front so the call resolver can follow it. + aliases = self._build_alias_map(tree, caller_file) for node in ast.walk(tree): if isinstance(node, ast.Call): - resolved = self._resolve_call_node(node, caller_file, caller_class, local_types) + resolved = self._resolve_call_node(node, caller_file, caller_class, local_types, aliases, self_aliases) if resolved: calls.add(resolved) # Higher-order-function callbacks: a function reference passed as an @@ -248,15 +256,146 @@ def _resolve_callback_args(self, node: ast.Call, caller_file: str) -> List[str]: callees.append(resolved) return callees + def _build_alias_map(self, tree: ast.AST, caller_file: str) -> Dict[str, str]: + """Map local names bound to a known function value -> that function's ID. + + Captures simple `name = other_name` bindings where `other_name` resolves + (same-file or via the global single-name index) to a user function. This + lets `fn = helper; fn()` recover the `helper` edge. Class/method targets + are out of scope (only ast.Name = ast.Name single-target assigns). + + SINGLE-UNCONDITIONAL-ASSIGNMENT GUARD: an alias is kept ONLY when the + name is bound exactly once AND at the function/module top level (a direct + statement of the body, not nested inside an If/For/While/Try/With or any + other block). A name bound 2+ times (last-write-wins) or bound inside a + conditional/loop/try/with is a "maybe" binding; resolving it would assert + a maybe as definite, so we drop it and let `x()` fall through to normal + resolution (no edge). Precision over recall. + """ + # Count EVERY assignment to each name anywhere in the body (any nesting, + # both Assign targets and AnnAssign), so a reassignment or a conditional + # rebinding disqualifies the name even if one binding is top-level. + assign_counts: Dict[str, int] = {} + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + for target in node.targets: + for name in self._assigned_names(target): + assign_counts[name] = assign_counts.get(name, 0) + 1 + elif isinstance(node, ast.AnnAssign) and node.value is not None: + for name in self._assigned_names(node.target): + assign_counts[name] = assign_counts.get(name, 0) + 1 + + # Only consider candidate aliases declared at the body's TOP LEVEL. + top_level = self._function_body_statements(tree) + aliases: Dict[str, str] = {} + for stmt in top_level: + if not isinstance(stmt, ast.Assign): + continue + if not isinstance(stmt.value, ast.Name): + continue + if len(stmt.targets) != 1: + continue + target = stmt.targets[0] + if not isinstance(target, ast.Name): + continue + if target.id == stmt.value.id: + continue + # GUARD: bound exactly once across the whole body (single + + # unconditional, because a conditional binding would push the + # top-level count higher OR not appear at top level at all). + if assign_counts.get(target.id, 0) != 1: + continue + resolved = self._resolve_simple_call(stmt.value.id, caller_file) + if resolved: + aliases[target.id] = resolved + return aliases + + @staticmethod + def _assigned_names(target: ast.AST): + """Yield the simple Name ids bound by an assignment target (recursively + through tuple/list unpacking).""" + if isinstance(target, ast.Name): + yield target.id + elif isinstance(target, (ast.Tuple, ast.List)): + for elt in target.elts: + yield from CallGraphBuilder._assigned_names(elt) + + @staticmethod + def _function_body_statements(tree: ast.AST) -> List[ast.stmt]: + """Return the top-level statements of the analysed function body. + + The extractor's `code` includes the `def` line, so a single function + parses to Module(body=[FunctionDef]); the real body is that def's body. + Fall back to the module body for bare module-level code. + """ + body = getattr(tree, 'body', []) + if len(body) == 1 and isinstance(body[0], (ast.FunctionDef, ast.AsyncFunctionDef)): + return list(body[0].body) + return list(body) + + def _resolve_local_function(self, func_name: str, caller_file: str) -> Optional[str]: + """Resolve a bare name to a same-file, non-method user function. + + Deliberately same-file ONLY (no global cross-file fallback) so a genuine + builtin call is never linked to an unrelated same-named function in + another file. + """ + for func_id in self.functions_by_file.get(caller_file, []): + func_data = self.functions.get(func_id, {}) + if func_data.get('name') == func_name and not func_data.get('class_name'): + return func_id + return None + + def _collect_self_aliases(self, tree: ast.AST) -> Set[str]: + """Local names single-bound to ``self``/``cls`` (e.g. ``obj = self``). + + Only single, unconditional bindings count: a name assigned more than + once — or ever rebound to something other than self/cls — is NOT an + alias, so no spurious self-method edge is created. + """ + assign_count: dict = {} + self_bound: Set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Assign): + for tgt in node.targets: + if isinstance(tgt, ast.Name): + assign_count[tgt.id] = assign_count.get(tgt.id, 0) + 1 + if isinstance(node.value, ast.Name) and node.value.id in ('self', 'cls'): + self_bound.add(tgt.id) + return {name for name in self_bound if assign_count.get(name) == 1} + def _resolve_call_node(self, node: ast.Call, caller_file: str, caller_class: Optional[str], - local_types: Optional[Dict[str, str]] = None) -> Optional[str]: - """Resolve an AST Call node to a function ID.""" + local_types: Optional[Dict[str, str]] = None, + aliases: Optional[Dict[str, str]] = None, + self_aliases: Optional[Set[str]] = None) -> Optional[str]: + """Resolve an AST Call node to a function ID. + + ``self_aliases`` is the set of receiver names that refer to the current + instance/class — always includes ``self``/``cls`` plus any local name + single-bound to them (``obj = self; obj.method()``). Defaults to the + bare ``{'self', 'cls'}`` so direct callers keep working. + """ func = node.func local_types = local_types or {} + if self_aliases is None: + self_aliases = {'self', 'cls'} # Simple function call: func_name(...) if isinstance(func, ast.Name): func_name = func.id + # Local function-value alias: `fn = helper; fn()` resolves to helper. + if aliases and func_name in aliases: + return aliases[func_name] + # A same-file user function with the same name as a stdlib + # module/builtin wins over the builtin filter: the call is + # unambiguously to the local definition. SCOPE is deliberately + # restricted to the caller's own file -- we do NOT use the global + # cross-file name index here, so a genuine builtin call (e.g. + # time()) is never linked to an unrelated same-named function in + # some other file. + local = self._resolve_local_function(func_name, caller_file) + if local: + return local if self._is_builtin(func_name): return None return self._resolve_simple_call(func_name, caller_file) @@ -266,14 +405,9 @@ def _resolve_call_node(self, node: ast.Call, caller_file: str, caller_class: Opt method_name = func.attr obj = func.value - # self.method(...) - same class - if isinstance(obj, ast.Name) and obj.id == 'self': - if caller_class: - return self._resolve_self_call(method_name, caller_file, caller_class) - return None - - # cls.method(...) - classmethod - if isinstance(obj, ast.Name) and obj.id == 'cls': + # self.method(...) / cls.method(...) — same class, including a local + # alias single-bound to self/cls (obj = self; obj.method()). + if isinstance(obj, ast.Name) and obj.id in self_aliases: if caller_class: return self._resolve_self_call(method_name, caller_file, caller_class) return None @@ -329,14 +463,34 @@ def _resolve_simple_call(self, func_name: str, caller_file: str) -> Optional[str return None def _resolve_self_call(self, method_name: str, caller_file: str, caller_class: str) -> Optional[str]: - """Resolve a self.method() call within a class.""" - class_key = f"{caller_file}:{caller_class}" - class_methods = self.methods_by_class.get(class_key, []) + """Resolve a self.method() call within a class or its (same-file) bases. - for func_id in class_methods: - func_data = self.functions.get(func_id, {}) - if func_data.get('name') == method_name: - return func_id + Walks the class first, then its base classes transitively (breadth-first, + cycle-guarded), so a method inherited from a base resolves. Base lookup + is restricted to classes defined in the caller's own file -- external + base classes aren't in our index, so they're left unresolved rather than + mis-linked. + """ + seen: Set[str] = set() + queue: List[str] = [caller_class] + while queue: + class_name = queue.pop(0) + if class_name in seen: + continue + seen.add(class_name) + + class_key = f"{caller_file}:{class_name}" + for func_id in self.methods_by_class.get(class_key, []): + func_data = self.functions.get(func_id, {}) + if func_data.get('name') == method_name: + return func_id + + class_data = self.classes.get(class_key, {}) + for base in class_data.get('bases', []): + # Only same-file base names are resolvable via our index. + base_name = base.split('.')[-1] + if base_name not in seen: + queue.append(base_name) return None @@ -474,14 +628,34 @@ def _resolve_import(self, import_path: str, func_name: str, caller_file: str, return reexported # Strategy 2: Check if import_path itself is a function + # The matched function's NAME must equal the called function name -- + # matching on parts[-1] (the module path tail) alone spuriously links + # e.g. `import alpha; alpha.run()` to a free function named `alpha`, + # ignoring that `run` (not `alpha`) was actually called. for func_id in self.functions: func_data = self.functions[func_id] - if func_data.get('name') == parts[-1]: - # Check if file path matches module path + if func_data.get('name') == func_name: + # Check if file path matches module path, by dotted COMPONENTS in EITHER + # direction: + # - imported module is a component-suffix of the file path — a repo-root + # prefix on the file (e.g. `src/pkg/auth.py` for `from pkg.auth ...`); OR + # - the file path is a component-suffix of the imported module — the + # repo-IS-the-package self-import (e.g. `auth.py` == `myapp/auth.py` + # recorded shallow via relative_to(repo_path), for `from myapp.auth ...`). + # Both are real layouts, so both are kept (recall). The OLD bare `endswith` + # ALSO matched ACROSS component boundaries — `'extra_utils'.endswith('utils')` + # bound `from utils import helper` to `extra_utils.py`. No module layout + # produces that, so those substring-crossing matches are the ONLY edges + # dropped. `not em` keeps the prior name-only behavior for relative/bare + # imports (`from . import foo`). file_path = func_data.get('file_path', '') module_path = file_path.replace('/', '.').replace('.py', '') expected_module = '.'.join(parts[:-1]) - if module_path.endswith(expected_module) or expected_module.endswith(module_path): + mp = module_path.split('.') + em = expected_module.split('.') if expected_module else [] + if (not em + or (len(em) <= len(mp) and mp[-len(em):] == em) + or (len(mp) <= len(em) and em[-len(mp):] == mp)): return func_id return None @@ -525,6 +699,12 @@ def _extract_calls_regex(self, code: str, caller_id: str) -> Set[str]: pattern = r'\b([a-zA-Z_][a-zA-Z0-9_]*)\s*\(' for match in re.finditer(pattern, code): func_name = match.group(1) + # Same as the AST path: a same-file user function named like a + # stdlib module/builtin still wins over the builtin filter. + local = self._resolve_local_function(func_name, caller_file) + if local: + calls.add(local) + continue if not self._is_builtin(func_name): resolved = self._resolve_simple_call(func_name, caller_file) if resolved: diff --git a/libs/openant-core/parsers/python/dataset_enhancer.py b/libs/openant-core/parsers/python/dataset_enhancer.py index 73efe062..42eac4e1 100644 --- a/libs/openant-core/parsers/python/dataset_enhancer.py +++ b/libs/openant-core/parsers/python/dataset_enhancer.py @@ -198,7 +198,7 @@ def resolve_recursive(current_file: Path, current_code: str, depth: int): dependencies.append({ "code": dep_code, "origin": { - "file_path": str(resolved_path.relative_to(self.repo_path)), + "file_path": resolved_path.relative_to(self.repo_path).as_posix(), "start_line": start, "end_line": end, "function_name": func_name @@ -213,7 +213,7 @@ def resolve_recursive(current_file: Path, current_code: str, depth: int): dependencies.append({ "code": dep_code, "origin": { - "file_path": str(current_file.relative_to(self.repo_path)), + "file_path": current_file.relative_to(self.repo_path).as_posix(), "start_line": start, "end_line": end, "function_name": func_ref diff --git a/libs/openant-core/parsers/python/function_extractor.py b/libs/openant-core/parsers/python/function_extractor.py index 37475e57..93f7ebc3 100644 --- a/libs/openant-core/parsers/python/function_extractor.py +++ b/libs/openant-core/parsers/python/function_extractor.py @@ -229,7 +229,12 @@ def classify_function(self, func_name: str, decorators: List[str], return 'constructor' if func_name.startswith('__') and func_name.endswith('__'): return 'dunder_method' - if '@property' in dec_str: + # @property getter, @.setter/.deleter, and @cached_property/ + # @functools.cached_property are all property accessors. Reuse the + # single _property_role predicate so classification can't drift from + # the qualified_name role-suffix logic (a literal '@property' match + # silently mislabels @cached_property as a plain method). + if self._property_role(decorators) is not None: return 'property' if '@staticmethod' in dec_str: return 'static_method' @@ -241,13 +246,19 @@ def classify_function(self, func_name: str, decorators: List[str], if 'middleware' in func_name.lower() or self._path_has_segment(file_path, 'middleware'): return 'middleware' - # Test functions. - # 'test' is matched as a substring here on purpose: test-file conventions use plural/affixed - # forms ('tests/' dir, 'test_*'/'*_test' files) that a whole-segment match would miss. The - # substring over-match (e.g. 'latest'/'contest') is the is_test_file family handled in its own - # scheduled units -- 'test' is NOT an entry-point type, so unlike 'views' it seeds no false - # reachability and is intentionally left as a substring test in this fix. - if func_name.startswith('test_') or 'test' in path_lower: + # Test functions. Match the file by PATH COMPONENT, not bare substring: + # 'test' in path_lower wrongly flags e.g. latest.py / contest.py / fastest.py. + # A real test file is one whose directory or filename is/starts-with 'test' + # (pytest's discovery convention: test_*.py / *_test.py / a tests/ dir). + path_parts = path_lower.replace('\\', '/').split('/') + filename = path_parts[-1] if path_parts else '' + is_test_path = ( + any(part == 'test' or part == 'tests' for part in path_parts[:-1]) + or filename.startswith('test_') + or filename.endswith('_test.py') + or filename == 'test.py' + ) + if func_name.startswith('test_') or is_test_path: return 'test' # Utility functions @@ -298,29 +309,250 @@ def extract_imports(self, tree: ast.AST, file_path: str) -> Dict[str, str]: return imports + def _property_role(self, decorators: List[str]) -> Optional[str]: + """Classify a property accessor from its decorators: getter | setter | + deleter | None. Match on the decorator's final dotted segment (TOKEN), + not a bare substring, so `@property`/`@cached_property`/ + `@functools.cached_property`/`@x.setter`/`@x.deleter` are recognized but + a method whose decorator merely CONTAINS the text (e.g. + `@some_property_validator`, `@app.property_route`) is NOT misclassified.""" + for d in decorators: + leaf = d.lstrip('@').split('(')[0].rsplit('.', 1)[-1] + if leaf == 'setter': + return 'setter' + if leaf == 'deleter': + return 'deleter' + for d in decorators: + leaf = d.lstrip('@').split('(')[0].rsplit('.', 1)[-1] + if leaf in ('property', 'cached_property'): + return 'getter' + return None + + def _store_function(self, func_id: str, func_data: Dict) -> str: + """Insert a function unit, disambiguating any residual func_id collision. + + Property accessors are already disambiguated by ROLE upstream (in + process_function, via the qualified_name), so they never collide here. + The residual cases are TRUE same-qualified-name duplicates -- two nested + defs of the same name in one scope, or a lambda sharing a name with a + def. Keying solely on qualified_name would let the second overwrite the + first (a recall loss), so disambiguate DETERMINISTICALLY by source line + (`#L`), never by emission order -- the canonical-unit choice must + be stable across edits. The earlier-in-source unit (parsed first) keeps + the clean id. + """ + if func_id not in self.functions: + self.functions[func_id] = func_data + return func_id + line = func_data.get('start_line', 0) + unique_id = f"{func_id}#L{line}" + n = 2 + while unique_id in self.functions: + unique_id = f"{func_id}#L{line}.{n}" + n += 1 + self.functions[unique_id] = func_data + return unique_id + + def _count_function(self, func_data: Dict, *, is_method: bool) -> None: + """Update statistics for a single emitted function/method unit.""" + self.stats['total_functions'] += 1 + if is_method: + self.stats['total_methods'] += 1 + else: + self.stats['standalone_functions'] += 1 + if func_data['is_async']: + self.stats['async_functions'] += 1 + unit_type = func_data['unit_type'] + self.stats['by_type'][unit_type] = self.stats['by_type'].get(unit_type, 0) + 1 + + # Block-statement containers whose bodies may hold def/class nodes the + # def/class-only recursion never reaches. Built defensively so Python + # versions lacking TryStar (<3.11) / Match (<3.10) don't raise. + _BLOCK_CONTAINERS = tuple(filter(None, ( + getattr(ast, _n, None) for _n in ( + 'If', 'For', 'AsyncFor', 'While', 'With', 'AsyncWith', + 'Try', 'TryStar', 'Match', + ) + ))) + + @staticmethod + def _block_bodies(stmt: ast.AST) -> List[list]: + """Every statement-list body of a block container (if/try/for/.../match).""" + bodies: List[list] = [] + for field in ('body', 'orelse', 'finalbody'): + v = getattr(stmt, field, None) + if isinstance(v, list): + bodies.append(v) + for handler in getattr(stmt, 'handlers', None) or []: # except arms + b = getattr(handler, 'body', None) + if isinstance(b, list): + bodies.append(b) + for case in getattr(stmt, 'cases', None) or []: # match arms + b = getattr(case, 'body', None) + if isinstance(b, list): + bodies.append(b) + return bodies + + def _descend_into_blocks(self, stmts: list, file_path: Path, content: str) -> None: + """Find def/class nodes inside block statements at ANY depth and emit them. + + A `def`/`class` inside an `if`/`try`/`for`/`while`/`with`/`match` block is + runtime-reachable (version guards, `try/except ImportError` fallbacks, + CBV `if/else` dispatchers) but the def/class-only recursion never entered + a block body, so it was dropped from both the inventory and the call + graph. This descends ONLY into block-container nodes — direct + `FunctionDef`/`ClassDef` children of a body are emitted by the caller, so + there is no double-processing (the two node sets are disjoint). Surfaced + defs reuse the existing keep-both (`#L`) machinery. + """ + for stmt in stmts: + if not isinstance(stmt, self._BLOCK_CONTAINERS): + continue + for body in self._block_bodies(stmt): + for child in body: + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): + self._process_function_tree(child, file_path, content, class_name=None) + elif isinstance(child, ast.ClassDef): + self._process_class_tree(child, file_path, content, outer_qualifier=None) + self._descend_into_blocks(body, file_path, content) + + def _process_function_tree(self, node: ast.AST, file_path: Path, content: str, + class_name: Optional[str] = None) -> None: + """Register a function and recurse into its body. + + Handles defs nested inside a function body (which the top-level child + iteration never reaches) and classes nested inside a function. Each + nested def is emitted as its own unit; nested classes are delegated to + process_class so their methods are extracted too. + """ + func_id, func_data = self.process_function(node, str(file_path), content, class_name) + self._store_function(func_id, func_data) + self._count_function(func_data, is_method=class_name is not None) + + # Recurse into the body: a def nested inside this function's body is + # never reached by the top-level / direct-method walks. + for child in node.body: + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): + # A def nested inside a function is a standalone (non-method) + # function in its own right; do not attribute it to a class. + self._process_function_tree(child, file_path, content, class_name=None) + elif isinstance(child, ast.ClassDef): + self._process_class_tree(child, file_path, content, outer_qualifier=None) + # defs/classes wrapped in a block inside this function's body + self._descend_into_blocks(node.body, file_path, content) + + def _process_class_tree(self, node: ast.ClassDef, file_path: Path, content: str, + outer_qualifier: Optional[str] = None) -> None: + """Register a class, its methods, and any classes nested within it. + + `outer_qualifier` is the dotted prefix of any enclosing class + (e.g. 'Outer' so an inner class method is keyed 'Outer.Inner.deep'). + """ + class_id, class_data, method_nodes = self.process_class( + node, str(file_path), content, outer_qualifier=outer_qualifier + ) + self.classes[class_id] = class_data + self.stats['total_classes'] += 1 + + qualified_class = f"{outer_qualifier}.{node.name}" if outer_qualifier else node.name + + for method_node, method_class_name in method_nodes: + # Methods may themselves contain nested defs -- recurse. + self._process_function_tree(method_node, file_path, content, class_name=method_class_name) + + # Recurse into nested classes so their methods are extracted. + for item in node.body: + if isinstance(item, ast.ClassDef): + self._process_class_tree(item, file_path, content, outer_qualifier=qualified_class) + # defs/classes wrapped in a block inside the class body (e.g. an + # `if TYPE_CHECKING:` block declaring conditional members). + self._descend_into_blocks(node.body, file_path, content) + + def extract_assigned_lambdas(self, tree: ast.AST, file_path: Path, content: str) -> None: + """Emit a function unit for each module-level `name = lambda ...`. + + Only FunctionDef/AsyncFunctionDef/ClassDef are recognised as units, so a + named lambda (a common handler / dispatch idiom) is invisible and calls + to it cannot resolve. Capture module-level single-target name bindings to + a lambda as functions. + """ + relative_path = file_path.relative_to(self.repo_path).as_posix() + for node in ast.iter_child_nodes(tree): + if not isinstance(node, ast.Assign): + continue + if not isinstance(node.value, ast.Lambda): + continue + for target in node.targets: + if not isinstance(target, ast.Name): + continue + name = target.id + func_id = f"{relative_path}:{name}" + params = [a.arg for a in node.value.args.args] + if node.value.args.vararg: + params.append(f"*{node.value.args.vararg.arg}") + for a in node.value.args.kwonlyargs: + params.append(a.arg) + if node.value.args.kwarg: + params.append(f"**{node.value.args.kwarg.arg}") + func_data = { + 'name': name, + 'qualified_name': name, + 'file_path': relative_path, + 'start_line': node.lineno, + 'end_line': getattr(node, 'end_lineno', node.lineno), + 'code': self.get_source_segment(content, node), + 'class_name': None, + 'decorators': [], + 'is_async': False, + 'parameters': params, + 'docstring': None, + 'unit_type': self.classify_function(name, [], None, relative_path), + 'is_lambda': True, + } + self._store_function(func_id, func_data) + self._count_function(func_data, is_method=False) + def process_function(self, node: ast.FunctionDef, file_path: str, content: str, class_name: Optional[str] = None) -> Dict: """Process a function definition and extract metadata.""" func_name = node.name - qualified_name = f"{class_name}.{func_name}" if class_name else func_name - - # Generate unique ID - relative_path = str(Path(file_path).relative_to(self.repo_path)) - func_id = f"{relative_path}:{qualified_name}" + relative_path = Path(file_path).relative_to(self.repo_path).as_posix() # Extract metadata decorators = self.extract_decorators(node) + + # @property getter, @x.setter and @x.deleter accessors all share the + # qualified name `Class.x`, which would collide into one func_id and let + # the setter overwrite the getter. Disambiguate by ROLE in the + # qualified_name (getter stays canonical `C.x`; setter -> `C.x.setter`, + # deleter -> `C.x.deleter`). This keeps func_id == path:qualified_name -- + # the invariant call_graph_builder relies on to reconstruct call targets + # -- and is order-independent (role is intrinsic, not emission position). + property_role = self._property_role(decorators) + qualified_name = f"{class_name}.{func_name}" if class_name else func_name + if property_role in ('setter', 'deleter'): + qualified_name = f"{qualified_name}.{property_role}" + + # Generate unique ID (after any role suffix) + func_id = f"{relative_path}:{qualified_name}" parameters = self.extract_parameters(node) docstring = self.get_docstring(node) code = self.get_source_segment(content, node) is_async = isinstance(node, ast.AsyncFunctionDef) unit_type = self.classify_function(func_name, decorators, class_name, relative_path) + # The captured `code` (get_source_segment) includes any decorator lines, + # so start_line must point at the first decorator, not the `def` line. + # Off-by-one for one decorator; off-by-N for stacked decorators. + start_line = node.lineno + if getattr(node, 'decorator_list', None): + start_line = min(start_line, min(d.lineno for d in node.decorator_list)) + func_data = { 'name': func_name, 'qualified_name': qualified_name, 'file_path': relative_path, - 'start_line': node.lineno, + 'start_line': start_line, 'end_line': getattr(node, 'end_lineno', node.lineno), 'code': code, 'class_name': class_name, @@ -329,14 +561,21 @@ def process_function(self, node: ast.FunctionDef, file_path: str, 'parameters': parameters, 'docstring': docstring[:500] if docstring else None, # Truncate long docstrings 'unit_type': unit_type, + 'property_role': property_role, } return func_id, func_data - def process_class(self, node: ast.ClassDef, file_path: str, content: str) -> Tuple[str, Dict, List[Tuple]]: - """Process a class definition and extract metadata.""" - class_name = node.name - relative_path = str(Path(file_path).relative_to(self.repo_path)) + def process_class(self, node: ast.ClassDef, file_path: str, content: str, + outer_qualifier: Optional[str] = None) -> Tuple[str, Dict, List[Tuple]]: + """Process a class definition and extract metadata. + + `outer_qualifier` is the dotted name of any enclosing class, so a class + nested inside another is keyed by its full path (e.g. 'Outer.Inner') and + its methods become 'Outer.Inner.method'. + """ + class_name = f"{outer_qualifier}.{node.name}" if outer_qualifier else node.name + relative_path = Path(file_path).relative_to(self.repo_path).as_posix() class_id = f"{relative_path}:{class_name}" # Extract base classes @@ -408,10 +647,13 @@ def extract_module_level_code(self, tree: ast.AST, content: str, lines = content.split('\n') total_lines = len(lines) - # Track which lines are covered by functions/classes + # Track which lines are covered by functions/classes. Walk the WHOLE + # tree (not just top-level children) so a def/class wrapped in a block + # (if/try/for/with/match) is covered too — otherwise its body, now its + # own unit, would also leak verbatim into this synthetic :__module__ text. covered_lines: Set[int] = set() - for node in ast.iter_child_nodes(tree): + for node in ast.walk(tree): if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): start_line = node.lineno end_line = getattr(node, 'end_lineno', start_line) @@ -478,7 +720,7 @@ def extract_module_level_code(self, tree: ast.AST, content: str, if not module_code: return None - relative_path = str(file_path.relative_to(self.repo_path)) + relative_path = file_path.relative_to(self.repo_path).as_posix() func_id = f"{relative_path}:__module__" # Determine start and end lines @@ -518,42 +760,26 @@ def process_file(self, file_path: Path) -> None: return self.stats['files_processed'] += 1 - relative_path = str(file_path.relative_to(self.repo_path)) + relative_path = file_path.relative_to(self.repo_path).as_posix() # Extract imports self.imports[relative_path] = self.extract_imports(tree, relative_path) - # Process top-level functions and classes + # Process top-level functions and classes. The tree helpers recurse so + # defs nested in function bodies and classes nested in classes/functions + # are also extracted (not just the direct children). for node in ast.iter_child_nodes(tree): if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - func_id, func_data = self.process_function(node, file_path, content) - self.functions[func_id] = func_data - self.stats['total_functions'] += 1 - self.stats['standalone_functions'] += 1 - if func_data['is_async']: - self.stats['async_functions'] += 1 + self._process_function_tree(node, file_path, content, class_name=None) + elif isinstance(node, ast.ClassDef): + self._process_class_tree(node, file_path, content, outer_qualifier=None) - # Track by type - unit_type = func_data['unit_type'] - self.stats['by_type'][unit_type] = self.stats['by_type'].get(unit_type, 0) + 1 + # defs/classes wrapped in a top-level block (version guard, try/except + # ImportError fallback, with-guarded handler, etc.). + self._descend_into_blocks(tree.body, file_path, content) - elif isinstance(node, ast.ClassDef): - class_id, class_data, method_nodes = self.process_class(node, file_path, content) - self.classes[class_id] = class_data - self.stats['total_classes'] += 1 - - # Process methods - for method_node, class_name in method_nodes: - func_id, func_data = self.process_function(method_node, file_path, content, class_name) - self.functions[func_id] = func_data - self.stats['total_functions'] += 1 - self.stats['total_methods'] += 1 - if func_data['is_async']: - self.stats['async_functions'] += 1 - - # Track by type - unit_type = func_data['unit_type'] - self.stats['by_type'][unit_type] = self.stats['by_type'].get(unit_type, 0) + 1 + # Module-level lambdas bound to a name (handler = lambda ...). + self.extract_assigned_lambdas(tree, file_path, content) # Extract module-level code module_result = self.extract_module_level_code(tree, content, file_path) diff --git a/libs/openant-core/parsers/ruby/call_graph_builder.py b/libs/openant-core/parsers/ruby/call_graph_builder.py index 9e1f7212..ad3ed3bf 100644 --- a/libs/openant-core/parsers/ruby/call_graph_builder.py +++ b/libs/openant-core/parsers/ruby/call_graph_builder.py @@ -190,12 +190,60 @@ def _extract_calls_from_code(self, code: str, caller_id: str) -> Set[str]: except Exception: return self._extract_calls_regex(code, caller_id) + # First pass: collect local-variable names (assignment LHS) and + # method-object bindings (`m = method(:sym)` / proc / lambda). These + # inform parenless-call precision [BUG 1] and `.call` resolution [BUG 31]. + # + # SINGLE-UNCONDITIONAL-ASSIGNMENT GUARD for method-object bindings: a + # binding is kept ONLY when the variable is assigned exactly once AND at + # the method/program top level (a direct statement of the body, not + # nested inside an `if`/`unless`/`while`/`case`/`begin`/block). A var + # assigned 2+ times (last-write-wins) or bound conditionally is a + # "maybe" binding; resolving its `.call` would assert a maybe as + # definite, so we drop it and let `m.call` go unresolved (no edge). + # `local_vars` (for parenless precision) is NOT narrowed -- any + # assignment target is still a local variable for the bare-identifier + # guard, regardless of how many times / where it is bound. + local_vars: Set[str] = set() + assign_counts: Dict[str, int] = {} + top_level_binding: Dict[str, str] = {} + stack = [tree.root_node] + while stack: + node = stack.pop() + if node.type == 'assignment': + lhs = node.children[0] if node.children else None + if lhs is not None and lhs.type == 'identifier': + var_name = self._node_str(lhs, code_bytes) + local_vars.add(var_name) + assign_counts[var_name] = assign_counts.get(var_name, 0) + 1 + if self._is_top_level_statement(node): + bound = self._method_object_target(node, code_bytes) + if bound: + top_level_binding[var_name] = bound + stack.extend(reversed(node.children)) + + # Keep a method-object binding only if it is single + unconditional: + # exactly one assignment to that name, and that binding is top-level. + method_object_bindings: Dict[str, str] = { + name: bound + for name, bound in top_level_binding.items() + if assign_counts.get(name, 0) == 1 + } + + # Second pass: resolve calls and bare (parenless) identifier calls. stack = [tree.root_node] while stack: node = stack.pop() if node.type == 'call': resolved = self._resolve_call_node(node, code_bytes, caller_file, - caller_class, caller_module, caller_method) + caller_class, caller_module, caller_method, + method_object_bindings) + if resolved: + calls.add(resolved) + elif node.type == 'identifier': + resolved = self._resolve_bare_identifier( + node, code_bytes, caller_file, caller_class, local_vars + ) if resolved: calls.add(resolved) elif node.type == 'super': @@ -209,15 +257,98 @@ def _extract_calls_from_code(self, code: str, caller_id: str) -> Set[str]: return calls + def _node_str(self, node, source: bytes) -> str: + return source[node.start_byte:node.end_byte].decode('utf-8', errors='replace') + + @staticmethod + def _is_top_level_statement(node) -> bool: + """True if `node` is a direct statement of a method/program body. + + Top-level means the parent is the method body (`body_statement`) or the + file program node. Anything nested in an `if`/`then`/`else`/`while`/ + `case`/`begin`/block has some other parent, so it is conditional/looped + and not a single-unconditional binding. + """ + parent = node.parent + return parent is not None and parent.type in ('body_statement', 'program') + + def _method_object_target(self, assignment_node, source: bytes) -> Optional[str]: + """If RHS is method(:sym)/proc/lambda binding a named method, return the name. + + Tracks `m = method(:helper)` so a later `m.call` resolves to `helper`. + """ + rhs = assignment_node.children[-1] if assignment_node.children else None + if rhs is None or rhs.type != 'call': + return None + callee = None + arg_list = None + for child in rhs.children: + if child.type == 'identifier' and callee is None: + callee = self._node_str(child, source) + elif child.type == 'argument_list': + arg_list = child + if callee != 'method' or arg_list is None: + return None + for arg in arg_list.children: + if arg.type in ('simple_symbol', 'symbol'): + sym = self._node_str(arg, source) + return sym.lstrip(':') + return None + + def _resolve_bare_identifier(self, node, source: bytes, caller_file: str, + caller_class: Optional[str], + local_vars: Set[str]) -> Optional[str]: + """Resolve a bare (parenless) identifier used as a call. + + Precision guard: only treat a bare identifier as a call when its name + is a KNOWN user function, it is NOT a Ruby builtin, and it is NOT a + local variable / assignment target in this method body. We also skip + identifiers that are a structural part of a call/method/assignment + (their own handlers cover those) to avoid double-counting. + """ + parent = node.parent + if parent is not None and parent.type in ( + 'call', 'method', 'singleton_method', 'assignment', 'method_call', + ): + return None + name = self._node_str(node, source) + if name in local_vars: + return None + if self._is_builtin(name): + return None + if name not in self.functions_by_name: + return None + return self._resolve_simple_call(name, caller_file, caller_class) + def _resolve_call_node(self, node, source: bytes, caller_file: str, caller_class: Optional[str], caller_module: Optional[str] = None, - caller_method: Optional[str] = None) -> Optional[str]: + caller_method: Optional[str] = None, + method_object_bindings: Optional[Dict[str, str]] = None + ) -> Optional[str]: """Resolve a tree-sitter call node to a function ID.""" # `super(args)` is a call node whose head is a `super` node, not an # identifier method name -- resolve it against the superclass. if any(c.type == 'super' for c in node.children): return self._resolve_super_call(caller_file, caller_class, caller_method) + + method_object_bindings = method_object_bindings or {} + + # Method-object call: `m = method(:helper)` then `m.call` resolves to the + # bound target [BUG 31]. The positional heuristic below mis-parses a + # lowercase-variable receiver (it captures the var as the method name), + # so detect this precisely via tree-sitter's named receiver/method fields. + recv_field = node.child_by_field_name('receiver') + meth_field = node.child_by_field_name('method') + if recv_field is not None and meth_field is not None: + recv_text = source[recv_field.start_byte:recv_field.end_byte].decode( + 'utf-8', errors='replace') + meth_text = source[meth_field.start_byte:meth_field.end_byte].decode( + 'utf-8', errors='replace') + if meth_text == 'call' and recv_text in method_object_bindings: + target_name = method_object_bindings[recv_text] + return self._resolve_simple_call(target_name, caller_file, caller_class) + # Extract method name method_name = None receiver = None @@ -262,7 +393,16 @@ def _resolve_call_node(self, node, source: bytes, caller_file: str, return self._resolve_class_call(receiver, target, caller_file) return None + # A user method may share a name with a Ruby builtin (e.g. render, log, + # open). Don't let the builtin filter drop a call that resolves to a + # user-defined function visible in scope. Scope = same class or same + # file ONLY (no global cross-file single-match) so a genuine builtin + # call isn't wired to an unrelated same-named user method elsewhere. if self._is_builtin(method_name): + if receiver is None and self._has_scoped_user_function( + method_name, caller_file, caller_class + ): + return self._resolve_simple_call(method_name, caller_file, caller_class) return None # self.method(...) - same class @@ -431,6 +571,25 @@ def _resolve_module_call(self, module_name: str, method_name: str, fallback = func_id return fallback + def _has_scoped_user_function(self, name: str, caller_file: str, + caller_class: Optional[str]) -> bool: + """True if `name` is a user function visible in the caller's scope. + + Scope is deliberately narrow (same class or same file). This lets a + user method that shadows a builtin name resolve, without globally + rescuing a genuine builtin call into an unrelated same-named method. + """ + if caller_class: + class_key = f"{caller_file}:{caller_class}" + for func_id in self.methods_by_class.get(class_key, []): + if self.functions.get(func_id, {}).get('name') == name: + return True + for func_id in self.functions_by_file.get(caller_file, []): + func_data = self.functions.get(func_id, {}) + if func_data.get('name') == name and not func_data.get('class_name'): + return True + return False + def _resolve_self_call(self, method_name: str, caller_file: str, caller_class: str) -> Optional[str]: """Resolve a self.method() call within a class.""" @@ -459,17 +618,39 @@ def _resolve_class_call(self, class_name: str, method_name: str, if func_data.get('name') == method_name: return func_id - # Check all files for the class + # Cross-file fallback. The previous behaviour linked the FIRST same-named + # class found ANYWHERE, which is unsound: it wired callers to unrelated + # files (and is wrong under same-name class collisions). Scope it: only + # accept a cross-file class when the caller's file require/require_relative + # resolves to the file that defines the class. for key, func_ids in self.methods_by_class.items(): - if key.endswith(f":{class_name}"): - for func_id in func_ids: - func_data = self.functions.get(func_id, {}) - if func_data.get('name') == method_name: - return func_id + if not key.endswith(f":{class_name}"): + continue + key_file = key.rsplit(':', 1)[0] + if not self._file_is_required_by(key_file, caller_file): + continue + for func_id in func_ids: + func_data = self.functions.get(func_id, {}) + if func_data.get('name') == method_name: + return func_id # The receiver may be a module (e.g. Utils.helper for a module_function). return self._resolve_module_call(class_name, method_name, caller_file) + def _file_is_required_by(self, target_file: str, caller_file: str) -> bool: + """True if caller_file has a require/require_relative resolving to target_file.""" + target_stem = target_file.replace('\\', '/').rsplit('/', 1)[-1] + if target_stem.endswith('.rb'): + target_stem = target_stem[:-len('.rb')] + file_imports = self.imports.get(caller_file, {}) + for import_name, import_type in file_imports.items(): + if import_type not in ('require', 'require_relative'): + continue + import_basename = import_name.replace('\\', '/').rsplit('/', 1)[-1] + if import_basename == target_stem: + return True + return False + def _extract_calls_regex(self, code: str, caller_id: str) -> Set[str]: """Fallback regex-based call extraction for unparseable code.""" calls = set() diff --git a/libs/openant-core/parsers/ruby/function_extractor.py b/libs/openant-core/parsers/ruby/function_extractor.py index 4ee61312..2d886683 100644 --- a/libs/openant-core/parsers/ruby/function_extractor.py +++ b/libs/openant-core/parsers/ruby/function_extractor.py @@ -138,12 +138,15 @@ def _classify_function(self, func_name: str, class_name: Optional[str], """ path_lower = file_path.lower() - if func_name == 'initialize': - return 'constructor' - + # is_singleton must be checked BEFORE the initialize/constructor branch: + # `def self.initialize` is a class-level singleton method, not the + # instance constructor (Ruby's constructor is the instance `initialize`). if is_singleton: return 'singleton_method' + if func_name == 'initialize': + return 'constructor' + # Non-public methods inside a class are helpers/callbacks, never # route handlers or callbacks-by-name, regardless of the enclosing # class. This must precede the controller branch so private params @@ -281,6 +284,16 @@ def _extract_imports(self, tree, source: bytes) -> Dict[str, str]: return imports + def _body_node(self, node): + """Return a node's body (`body` field, else a `body_statement` child).""" + body_node = node.child_by_field_name('body') + if body_node is None: + for child in node.children: + if child.type == 'body_statement': + body_node = child + break + return body_node + def _extract_functions_from_tree(self, tree, source: bytes, file_path: Path, relative_path: str) -> None: """Extract all method definitions from a parsed tree. @@ -303,12 +316,27 @@ def _extract_functions_from_tree(self, tree, source: bytes, file_path: Path, node, source, relative_path, class_name, module_name, is_singleton=False, visibility=vis_state[0], ) + # A nested `def` lives in the method's body -- keep traversing so + # methods defined inside another method are not lost. Nested defs + # inherit the enclosing class but default to public visibility. + body_node = self._body_node(node) + if body_node: + nested_vis = ['public'] + for child in reversed(body_node.children): + stack.append((child, class_name, module_name, nested_vis)) + continue elif node.type == 'singleton_method': self._process_method_node( node, source, relative_path, class_name, module_name, is_singleton=True, visibility=vis_state[0], ) + body_node = self._body_node(node) + if body_node: + nested_vis = ['public'] + for child in reversed(body_node.children): + stack.append((child, class_name, module_name, nested_vis)) + continue elif node.type == 'alias': # `alias new old` keyword form. @@ -364,7 +392,13 @@ def _extract_functions_from_tree(self, tree, source: bytes, file_path: Path, elif node.type == 'class': # Extract class name name_node = node.child_by_field_name('name') - new_class_name = self._node_text(name_node, source) if name_node else None + local_class_name = self._node_text(name_node, source) if name_node else None + # Compose with the enclosing class so a nested class keeps its + # outer qualifier, e.g. `Outer::Inner`. + if local_class_name and class_name: + new_class_name = f"{class_name}::{local_class_name}" + else: + new_class_name = local_class_name # Extract superclass superclass = None @@ -512,7 +546,7 @@ def _emit_synthetic_method(self, node, source: bytes, relative_path: str, qualified_name = name func_id = f"{relative_path}:{qualified_name}" - self.functions[func_id] = { + self._store_function(func_id, { 'name': name, 'qualified_name': qualified_name, 'file_path': relative_path, @@ -525,7 +559,7 @@ def _emit_synthetic_method(self, node, source: bytes, relative_path: str, 'is_singleton': False, 'visibility': visibility, 'unit_type': unit_type, - } + }) self.stats['total_functions'] += 1 if class_name: self.stats['total_methods'] += 1 @@ -533,6 +567,33 @@ def _emit_synthetic_method(self, node, source: bytes, relative_path: str, self.stats['standalone_functions'] += 1 self.stats['by_type'][unit_type] = self.stats['by_type'].get(unit_type, 0) + 1 + + def _store_function(self, func_id: str, func_data: dict) -> str: + """Insert a function unit, keeping BOTH on a same-(file,name) collision. + + Ruby `def` executes when reached, so same-name defs in mutually-exclusive + conditional branches (`if/else`) are both runtime-reachable depending on + the condition, and the EARLIER branch may be the live one. A plain + keep-last store let the later (possibly dead) branch overwrite the + earlier (possibly live) one — a silent false negative, confirmed against + the real `ruby` interpreter. Keep BOTH via a deterministic `#L` + suffix (earlier-in-source keeps the clean id), mirroring the Python + extractor's `_store_function`. Unconditional reopening is last-wins at + runtime; keeping both there is the same benign tradeoff Python accepts. + Collision-only: a unique name keeps its byte-identical `path:name` id. + """ + if func_id not in self.functions: + self.functions[func_id] = func_data + return func_id + line = func_data.get('start_line', 0) + unique_id = f"{func_id}#L{line}" + n = 2 + while unique_id in self.functions: + unique_id = f"{func_id}#L{line}.{n}" + n += 1 + self.functions[unique_id] = func_data + return unique_id + def _process_method_node(self, node, source: bytes, relative_path: str, class_name: Optional[str], module_name: Optional[str], is_singleton: bool, visibility: str = 'public') -> None: @@ -575,7 +636,7 @@ def _process_method_node(self, node, source: bytes, relative_path: str, 'unit_type': unit_type, } - self.functions[func_id] = func_data + self._store_function(func_id, func_data) self.stats['total_functions'] += 1 if class_name: diff --git a/libs/openant-core/parsers/ruby/test_pipeline.py b/libs/openant-core/parsers/ruby/test_pipeline.py index cb61d151..228e3b2c 100644 --- a/libs/openant-core/parsers/ruby/test_pipeline.py +++ b/libs/openant-core/parsers/ruby/test_pipeline.py @@ -48,7 +48,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) from utilities.file_io import open_utf8, read_json, run_utf8, write_json from utilities.context_enhancer import ContextEnhancer -from utilities.agentic_enhancer import EntryPointDetector, ReachabilityAnalyzer +from utilities.agentic_enhancer import EntryPointDetector, ReachabilityAnalyzer, blackout_warning, library_seed_ids # Local imports from repository_scanner import RepositoryScanner @@ -78,7 +78,8 @@ def __init__( processing_level: ProcessingLevel = ProcessingLevel.ALL, skip_tests: bool = False, depth: int = 3, - name: str = None + name: str = None, + library_mode: bool = False ): self.repo_path = os.path.abspath(repo_path) self.output_dir = output_dir or os.path.join(os.path.dirname(__file__), 'test_output') @@ -89,6 +90,7 @@ def __init__( self.skip_tests = skip_tests self.depth = depth self.dataset_name = name + self.library_mode = library_mode # Pipeline artifacts self.scan_results_file = None @@ -283,6 +285,11 @@ def apply_reachability_filter(self) -> bool: detector = EntryPointDetector(normalized_functions, call_graph) self.entry_points = detector.detect_entry_points() + # Library-mode: seed the exported public API (a library has no + # main/route/CLI marker). Union-only — never drops a real entry point. + if self.library_mode: + self.entry_points = self.entry_points | library_seed_ids(normalized_functions) + # Build reachability reachability = ReachabilityAnalyzer( functions=normalized_functions, @@ -314,6 +321,13 @@ def apply_reachability_filter(self) -> bool: "reduction_percentage": round((1 - len(filtered_units) / original_count) * 100, 1) if original_count > 0 else 0 } + _blackout = blackout_warning(detector.entry_point_details, original_count, + len(filtered_units), + library_mode=getattr(self, "library_mode", False)) + if _blackout: + dataset["metadata"]["reachability_filter"]["warning"] = _blackout + print(f" [Warning] {_blackout}", file=sys.stderr) + write_json(self.dataset_file, dataset) elapsed = (datetime.now() - start_time).total_seconds() @@ -1003,6 +1017,11 @@ def main(): default=None, help='Dataset name (default: derived from repo path)' ) + parser.add_argument( + '--library-mode', + action='store_true', + help='Seed the exported public API as entry points (for libraries with no main/route/CLI)' + ) args = parser.parse_args() @@ -1024,7 +1043,8 @@ def main(): processing_level=processing_level, skip_tests=args.skip_tests, depth=args.depth, - name=args.name + name=args.name, + library_mode=args.library_mode ) results = pipeline.run_full_pipeline() diff --git a/libs/openant-core/parsers/zig/call_graph_builder.py b/libs/openant-core/parsers/zig/call_graph_builder.py index bca6dda5..ee3aa3c9 100644 --- a/libs/openant-core/parsers/zig/call_graph_builder.py +++ b/libs/openant-core/parsers/zig/call_graph_builder.py @@ -152,14 +152,20 @@ def build_call_graph(self) -> None: name_to_ids = self._build_name_index() + # Build per-file simple const fn-alias bindings (`const f = handler;`) + # so that a later `f()` resolves to `handler`. + alias_to_target = self._build_alias_index(name_to_ids) + for func_id, func_info in self.functions.items(): code = func_info.get("code", "") file_path = func_info.get("file_path", "") - calls = self._find_calls_in_code(code) + calls = self._find_calls_in_code(code, file_path) for call_name in calls: - resolved_ids = self._resolve_call(call_name, file_path, name_to_ids) + resolved_ids = self._resolve_call( + call_name, file_path, name_to_ids, alias_to_target + ) for resolved_id in resolved_ids: if resolved_id != func_id: # No self-calls if resolved_id not in call_graph[func_id]: @@ -261,7 +267,59 @@ def _build_name_index(self) -> Dict[str, List[str]]: return name_to_ids - def _find_calls_in_code(self, code: str) -> Set[str]: + def _build_alias_index( + self, name_to_ids: Dict[str, List[str]] + ) -> Dict[str, Dict[str, str]]: + """Index simple const fn-aliases per file: `const f = handler;` -> {f: handler}. + + Only bindings whose right-hand side is a bare identifier naming a known + function are tracked (a genuine fn alias), so arbitrary const dataflow + (`const x = 1;`) is ignored. Scoped per file to avoid cross-file leaks. + """ + alias_to_target: Dict[str, Dict[str, str]] = defaultdict(dict) + + for func_info in self.functions.values(): + file_path = func_info.get("file_path", "") + code = func_info.get("code", "") + if not code: + continue + try: + tree = self.parser.parse(code.encode("utf-8")) + except Exception: + continue + self._collect_aliases_from_node( + tree.root_node, + code.encode("utf-8"), + name_to_ids, + alias_to_target[file_path], + ) + + return alias_to_target + + def _collect_aliases_from_node( + self, + node: Node, + source: bytes, + name_to_ids: Dict[str, List[str]], + aliases: Dict[str, str], + ) -> None: + """Collect `const = ;` bindings from a parse tree.""" + if node.type in ("variable_declaration", "VarDecl"): + ident_children = [ + c for c in node.children if c.type in ("identifier", "IDENTIFIER") + ] + # A simple alias is exactly: const = ; + if len(ident_children) == 2: + alias_name = self._get_node_text(ident_children[0], source) + target_name = self._get_node_text(ident_children[1], source) + # Only record when the target is a known function name. + if alias_name and target_name in name_to_ids: + aliases[alias_name] = target_name + + for child in node.children: + self._collect_aliases_from_node(child, source, name_to_ids, aliases) + + def _find_calls_in_code(self, code: str, caller_file: str = "") -> Set[str]: """Find all function calls in a code snippet.""" calls = set() @@ -272,11 +330,32 @@ def _find_calls_in_code(self, code: str) -> Set[str]: # Fallback to regex-based extraction calls = self._find_calls_with_regex(code) - # Filter out builtins - calls = {c for c in calls if c not in self.ZIG_BUILTINS and not c.startswith("@")} + # Filter out builtins, but NEVER filter a name that a same-file user + # function actually defines. A user fn whose name collides with a + # ZIG_BUILTINS entry (e.g. `expect`) must keep its edge. Scope the + # shadow check to the caller's own file so a builtin call is not + # spuriously linked to an unrelated same-named user fn elsewhere. + shadowing = self._same_file_function_names(caller_file) + calls = { + c + for c in calls + if c in shadowing or (c not in self.ZIG_BUILTINS and not c.startswith("@")) + } return calls + def _same_file_function_names(self, caller_file: str) -> Set[str]: + """Names of user functions defined in `caller_file` (same-file scope).""" + if not caller_file: + return set() + names: Set[str] = set() + for func_info in self.functions.values(): + if func_info.get("file_path") == caller_file: + name = func_info.get("name", "") + if name: + names.add(name) + return names + def _extract_calls_from_node( self, node: Node, source: bytes, calls: Set[str] ) -> None: @@ -288,9 +367,16 @@ def _extract_calls_from_node( if callee is not None and callee.type in ("identifier", "IDENTIFIER"): calls.add(self._get_node_text(callee, source)) elif callee is not None and callee.type in ("field_expression", "field_access"): + # The method name is the trailing identifier child. Prefer that + # over text-splitting, which is brittle when the receiver itself + # contains punctuation (e.g. `C{}.m`). + method_name = None + for sub in callee.children: + if sub.type in ("identifier", "IDENTIFIER"): + method_name = self._get_node_text(sub, source) text = self._get_node_text(callee, source) - calls.add(text.split(".")[-1]) # trailing member (method / func name) - calls.add(text) # also the full dotted form + calls.add(method_name if method_name else text.split(".")[-1]) + calls.add(text) # also the full dotted form elif node.type == "builtin_function": # @call(.modifier, realFn, argsTuple): the wrapped function is the real call target; # other @builtins are filtered out downstream. @@ -352,6 +438,7 @@ def _resolve_call( call_name: str, caller_file: str, name_to_ids: Dict[str, List[str]], + alias_to_target: Dict[str, Dict[str, str]] | None = None, ) -> List[str]: """ Resolve a call name to function ID(s). @@ -361,6 +448,13 @@ def _resolve_call( 2. Imported files 3. Unique name match """ + # Resolve a same-file const fn-alias (`const f = handler; f()`) to its + # target function name before looking up candidates. + if alias_to_target is not None: + target = alias_to_target.get(caller_file, {}).get(call_name) + if target is not None: + call_name = target + candidates = name_to_ids.get(call_name, []) if not candidates: diff --git a/libs/openant-core/parsers/zig/function_extractor.py b/libs/openant-core/parsers/zig/function_extractor.py index 7e5c2f57..93af90b3 100644 --- a/libs/openant-core/parsers/zig/function_extractor.py +++ b/libs/openant-core/parsers/zig/function_extractor.py @@ -131,6 +131,15 @@ def _walk_node( if func_info: func_id = f"{file_path}:{func_info['qualified_name']}" functions[func_id] = func_info + # Zig's generic-container idiom is a type-returning function: + # `fn List(comptime T: type) type { return struct { fn push() ... }; }`. + # The returned struct is anonymous in the AST (not a `const Name = + # struct {...}` variable_declaration), so without this its methods would + # recurse with current_struct unchanged and be emitted as bare top-level + # functions. Thread the function name as the struct context so they + # qualify as List.push and distinct containers' methods don't collide. + if self._returns_type(node, source): + child_struct = func_info["name"] elif node.type == "variable_declaration": # `const Foo = struct { ... };` -- a named struct/enum definition. @@ -206,6 +215,23 @@ def _extract_function( "unit_type": unit_type, } + def _returns_type(self, node: Node, source: bytes) -> bool: + """True if a function_declaration's return type is the builtin `type` — Zig's + generic-container idiom (`fn Foo(...) type { return struct {...} }`). + + The return type is the function_declaration's direct child that follows the + `parameters` node (a `builtin_type`). This deliberately inspects only direct + children, so the `type` inside a `comptime T: type` parameter (nested under + `parameters`) is not mistaken for the return type. + """ + seen_params = False + for child in node.children: + if child.type in ("parameters", "ParamDeclList"): + seen_params = True + elif seen_params and child.type == "builtin_type": + return self._get_node_text(child, source).strip() == "type" + return False + def _extract_parameters(self, node: Node, source: bytes) -> List[str]: """Extract parameter names from a parameter list node.""" params = [] @@ -228,7 +254,8 @@ def _extract_struct_from_var_decl( if child.type in ("identifier", "IDENTIFIER"): if name is None: name = self._get_node_text(child, source) - elif child.type == "struct_declaration": + elif child.type in ("struct_declaration", "enum_declaration", + "union_declaration", "opaque_declaration"): is_struct = True if name and is_struct: @@ -260,8 +287,11 @@ def _classify_function(self, name: str, file_path: str) -> str: """Classify the function type based on name and context.""" name_lower = name.lower() - # Test functions - if name_lower.startswith("test") or "_test" in name_lower: + # Test functions. Anchor on the underscore-delimited test convention + # (`test_foo`, `foo_test`, or a bare `test`). A camelCase identifier + # that merely starts with "test" (e.g. `testConnection`) is an ordinary + # function, not a zig `test "..." {}` block. + if name_lower == "test" or name_lower.startswith("test_") or name_lower.endswith("_test"): return "test" # Init/constructor patterns diff --git a/libs/openant-core/parsers/zig/test_pipeline.py b/libs/openant-core/parsers/zig/test_pipeline.py index bbde2193..c8a11930 100644 --- a/libs/openant-core/parsers/zig/test_pipeline.py +++ b/libs/openant-core/parsers/zig/test_pipeline.py @@ -49,6 +49,11 @@ def main(): "--skip-tests", action="store_true", help="Skip test files and functions" ) parser.add_argument("--name", help="Dataset name (defaults to repo directory name)") + parser.add_argument( + "--library-mode", + action="store_true", + help="Seed the exported public API as entry points (for libraries with no main/route/CLI)", + ) parser.add_argument( "--dependency-depth", type=int, @@ -129,7 +134,8 @@ def main(): # Apply processing level filters if args.processing_level != "all": call_graph_output = apply_processing_filter( - call_graph_output, args.processing_level, str(repo_path) + call_graph_output, args.processing_level, str(repo_path), + library_mode=args.library_mode, ) print( f" After {args.processing_level} filter: {len(call_graph_output['functions'])} functions", @@ -161,7 +167,7 @@ def main(): def apply_processing_filter( - call_graph_output: dict, level: str, repo_path: str + call_graph_output: dict, level: str, repo_path: str, library_mode: bool = False ) -> dict: """ Apply processing level filters to reduce the function set. @@ -173,21 +179,22 @@ def apply_processing_filter( - exploitable: Filter to reachable + CodeQL + LLM-classified exploitable """ if level == "reachable": - return apply_reachability_filter(call_graph_output, repo_path) + return apply_reachability_filter(call_graph_output, repo_path, library_mode=library_mode) elif level == "codeql": # First apply reachability, then would filter by CodeQL results - filtered = apply_reachability_filter(call_graph_output, repo_path) + filtered = apply_reachability_filter(call_graph_output, repo_path, library_mode=library_mode) # CodeQL filtering would be applied here if results exist return filtered elif level == "exploitable": # Apply all filters - filtered = apply_reachability_filter(call_graph_output, repo_path) + filtered = apply_reachability_filter(call_graph_output, repo_path, library_mode=library_mode) # CodeQL + LLM filtering would be applied here return filtered return call_graph_output -def apply_reachability_filter(call_graph_output: dict, repo_path: str) -> dict: +def apply_reachability_filter(call_graph_output: dict, repo_path: str, + library_mode: bool = False) -> dict: """Filter to functions reachable from entry points. Uses the real EntryPointDetector / ReachabilityAnalyzer contract, matching @@ -201,7 +208,7 @@ def apply_reachability_filter(call_graph_output: dict, repo_path: str) -> dict: --processing-level reachable. """ try: - from utilities.agentic_enhancer.entry_point_detector import EntryPointDetector + from utilities.agentic_enhancer.entry_point_detector import EntryPointDetector, blackout_warning, library_seed_ids from utilities.agentic_enhancer.reachability_analyzer import ReachabilityAnalyzer except ImportError: print( @@ -219,6 +226,11 @@ def apply_reachability_filter(call_graph_output: dict, repo_path: str) -> dict: detector = EntryPointDetector(functions, call_graph) entry_points = detector.detect_entry_points() + # Library-mode: seed the exported public API (a library has no main/route/CLI + # marker). Union-only — never drops a structurally-detected entry point. + if library_mode: + entry_points = entry_points | library_seed_ids(functions) + # Compute the reachable set via reverse-BFS from the entry points. analyzer = ReachabilityAnalyzer( functions=functions, @@ -247,6 +259,11 @@ def apply_reachability_filter(call_graph_output: dict, repo_path: str) -> dict: if k in reachable } + _blackout = blackout_warning(detector.entry_point_details, len(functions), + len(filtered_functions), library_mode=library_mode) + if _blackout: + print(f" [Warning] {_blackout}", file=sys.stderr) + return result diff --git a/libs/openant-core/prompts/verification_prompts.py b/libs/openant-core/prompts/verification_prompts.py index c37a38d9..8ac826a0 100644 --- a/libs/openant-core/prompts/verification_prompts.py +++ b/libs/openant-core/prompts/verification_prompts.py @@ -35,7 +35,7 @@ def get_verification_system_prompt(app_context: "ApplicationContext" = None) -> """ base_prompt = VERIFICATION_SYSTEM_PROMPT - if app_context and not app_context.requires_remote_trigger: + if app_context and app_context.suppress_local_only(): base_prompt += """ IMPORTANT: This is a CLI tool or library. The user running this code has local filesystem access. @@ -74,7 +74,7 @@ def format_app_context_for_verification(app_context: "ApplicationContext") -> st lines.append(f"- {item}") lines.append("") - if not app_context.requires_remote_trigger: + if app_context.suppress_local_only(): lines.append("**CRITICAL:** This is a CLI tool/library. Users have local filesystem access.") lines.append("A vulnerability requires a REMOTE attacker to exploit it.") lines.append("If the 'attack' requires running CLI commands locally, it's NOT a vulnerability.") @@ -149,7 +149,7 @@ def get_verification_prompt( {fence}""" # Adjust attacker description based on app context - if app_context and not app_context.requires_remote_trigger: + if app_context and app_context.suppress_local_only(): attacker_description = """You are an attacker on the internet. You have a browser and nothing else. No server access, no admin credentials, no ability to modify files on the server, and NO ABILITY TO RUN CLI COMMANDS. diff --git a/libs/openant-core/prompts/vulnerability_analysis.py b/libs/openant-core/prompts/vulnerability_analysis.py index 130989be..398cb89f 100644 --- a/libs/openant-core/prompts/vulnerability_analysis.py +++ b/libs/openant-core/prompts/vulnerability_analysis.py @@ -60,7 +60,7 @@ def format_app_context_for_prompt(app_context: "ApplicationContext") -> str: lines.append(f"- {item}") lines.append("") - if not app_context.requires_remote_trigger: + if app_context.suppress_local_only(): lines.append("**IMPORTANT:** This is a CLI tool/library. Users running this code have local access.") lines.append("Only flag vulnerabilities that could be exploited by a REMOTE attacker, not by local users.") lines.append("") @@ -155,7 +155,7 @@ def get_analysis_prompt( >>> END OF TARGET FUNCTION <<<""" # Build the appropriate questions based on whether we have app context - if app_context and not app_context.requires_remote_trigger: + if app_context and app_context.suppress_local_only(): input_question = """2. **Where does input come from and who controls it?** - Remote attackers (HTTP requests, external APIs)? → potential concern - Local users running the CLI/library? → NOT an attack vector (they have local access) @@ -228,7 +228,7 @@ def get_system_prompt(app_context: "ApplicationContext" = None) -> str: """ base_prompt = STAGE1_SYSTEM_PROMPT - if app_context and not app_context.requires_remote_trigger: + if app_context and app_context.suppress_local_only(): base_prompt += """ IMPORTANT: This is a CLI tool or library. The user running this code has local filesystem access. diff --git a/libs/openant-core/tests/parsers/c/test_c_collision_discriminator.py b/libs/openant-core/tests/parsers/c/test_c_collision_discriminator.py new file mode 100644 index 00000000..9459fb06 --- /dev/null +++ b/libs/openant-core/tests/parsers/c/test_c_collision_discriminator.py @@ -0,0 +1,108 @@ +"""Bug 1 regression: same-(file,name) C/C++ functions must not collapse to one +`func_id`. Two trigger families, two policies (see reachability-bugs.md Bug 1): + + * C++ overloads — `area(int)` and `area(int,int)` are BOTH real. They must + survive as two distinct func_ids, and each must keep its OWN call edges + (the conflation bug routed a call to overload-1 into overload-2's callee). + The discriminator is folded into the func_id KEY only; the `name` field + stays bare (`area`) so name-based call resolution still finds both. + * `#ifdef`/`#else` redefinition — same name, same signature, different body. + Only one is the compiled implementation; keep a SINGLE node and prefer the + larger body (the `#else` stub is shorter regardless of source order). + +Plus a regression guard: a uniquely-named function's id is byte-identical to the +pre-fix `path:name` form (the collision-only contract — non-colliding ids never +change, so the 299 hardcoded id literals across the suite are untouched). +""" +import sys +import tempfile +from pathlib import Path + +import pytest + +_CORE = Path(__file__).resolve().parents[3] +if str(_CORE) not in sys.path: + sys.path.insert(0, str(_CORE)) + +pytest.importorskip("tree_sitter_c") + +from parsers.c.function_extractor import FunctionExtractor # noqa: E402 +from parsers.c.call_graph_builder import CallGraphBuilder # noqa: E402 + + +def _extract(filename: str, source: str) -> dict: + repo = Path(tempfile.mkdtemp()).resolve() + fp = repo / filename + fp.write_text(source) + ex = FunctionExtractor(str(repo)) + ex.process_file(fp) + return ex.functions + + +def test_cpp_overloads_both_survive(): + fns = _extract("ovl.cpp", + "int area(int s){return s*s;}\n" + "int area(int w,int h){return w*h;}\n") + areas = [fid for fid, d in fns.items() if d["name"] == "area"] + assert len(areas) == 2, f"both overloads must survive; got {areas}" + assert len({fid for fid in areas}) == 2, "overload func_ids must be distinct" + + +def test_overload_ids_are_colon_free_in_name_part(): + # The discriminator must not introduce a colon into the name part of the id + # (downstream diff_filter / repository_index rsplit on ':'). + fns = _extract("ovl.cpp", + "int f(int a){return a;}\n" + "int f(const char* s,int n){return n;}\n") + for fid, d in fns.items(): + if d["name"] != "f": + continue + name_part = fid.split(":", 1)[1] # everything after the path colon + assert ":" not in name_part, f"colon leaked into discriminated id: {fid!r}" + + +def test_ifdef_else_keeps_larger_body_not_stub(): + # tree-sitter parses BOTH preprocessor arms; the #else stub comes last and, + # pre-fix, overwrote the real implementation. Prefer the larger body. + src = ( + "#ifdef FEATURE\n" + "int run(int n){int t=0;for(int i=0;i the dotted location where create_unit() should place it. + * A small reusable `get_path(obj, dotted)` walks that location. + * One parametrized test drives the REAL extractor + REAL UnitGenerator on a + function exercising every flag (static, inline) and asserts each contracted + field is present at its location and round-trips the producer's value. + +To add a future field to the contract: extend FIELD_CONTRACT. If create_unit() +forgets to carry it, this test fails — no per-field hand audit needed. +""" + +import sys +from pathlib import Path + +import pytest + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +from parsers.c.function_extractor import FunctionExtractor +from parsers.c.unit_generator import UnitGenerator + + +# Source exercising the contracted flags: `static inline` => is_static=True, +# is_inline=True, is_exported=False, plus return_type/parameters populated. +CONTRACT_SRC = "static inline int add(int a, int b) {\n return a + b;\n}\n" +TARGET_ID = "m.c:add" + +# extractor func_data key -> dotted location in the assembled unit. +# This is the field-contract the consumer (create_unit) must honor. +FIELD_CONTRACT = { + "is_static": "metadata.is_static", + "is_exported": "metadata.is_exported", + "is_inline": "metadata.is_inline", + "return_type": "metadata.return_type", + "parameters": "metadata.parameters", + "unit_type": "unit_type", + "name": "code.primary_origin.function_name", + "file_path": "code.primary_origin.file_path", + "class_name": "code.primary_origin.class_name", + "start_line": "code.primary_origin.start_line", + "end_line": "code.primary_origin.end_line", +} + +_MISSING = object() + + +def get_path(obj, dotted): + """Walk a dotted path through nested dicts; return _MISSING if any hop absent.""" + cur = obj + for part in dotted.split("."): + if not isinstance(cur, dict) or part not in cur: + return _MISSING + cur = cur[part] + return cur + + +@pytest.fixture(scope="module") +def extracted_and_unit(tmp_path_factory): + tmp_path = tmp_path_factory.mktemp("c_schema") + (tmp_path / "m.c").write_text(CONTRACT_SRC) + + extractor = FunctionExtractor(str(tmp_path)) + functions = extractor.extract_all(files=["m.c"])["functions"] + assert TARGET_ID in functions, f"target not extracted; got {list(functions)}" + + gen = UnitGenerator({ + "repository": str(tmp_path), + "functions": functions, + "call_graph": {}, + "reverse_call_graph": {}, + }) + unit = gen.create_unit(TARGET_ID, functions[TARGET_ID]) + return functions[TARGET_ID], unit + + +# Parallel consumer: generate_analyzer_output() re-emits a camelCase schema. +# It is the SECOND place BUG-29's field could drift; guard it too. +# extractor snake_case key -> analyzer_output camelCase key. +ANALYZER_CONTRACT = { + "name": "name", + "unit_type": "unitType", + "code": "code", + "file_path": "filePath", + "start_line": "startLine", + "end_line": "endLine", + "is_static": "isStatic", + "is_exported": "isExported", + "is_inline": "isInline", + "return_type": "returnType", + "parameters": "parameters", + "class_name": "className", +} + + +@pytest.mark.parametrize("producer_key,camel_key", sorted(ANALYZER_CONTRACT.items())) +def test_analyzer_output_contract_carried(extracted_and_unit, producer_key, camel_key): + """generate_analyzer_output() must re-emit every contracted field (camelCase).""" + func_data, _unit = extracted_and_unit + gen = UnitGenerator({ + "repository": "", + "functions": {TARGET_ID: func_data}, + "call_graph": {}, + "reverse_call_graph": {}, + }) + out = gen.generate_analyzer_output()["functions"][TARGET_ID] + assert camel_key in out, ( + f"field '{producer_key}' dropped from analyzer_output: " + f"expected camelCase key '{camel_key}'; keys = {sorted(out)}" + ) + assert out[camel_key] == func_data[producer_key] + + +@pytest.mark.parametrize("producer_key,unit_location", sorted(FIELD_CONTRACT.items())) +def test_field_contract_carried(extracted_and_unit, producer_key, unit_location): + """Each extractor-produced field must appear at its contracted unit location.""" + func_data, unit = extracted_and_unit + + assert producer_key in func_data, ( + f"contract assumes extractor produces '{producer_key}', but it did not " + f"(producer keys = {sorted(func_data)})" + ) + + value = get_path(unit, unit_location) + assert value is not _MISSING, ( + f"field '{producer_key}' dropped at unit assembly: " + f"expected at unit location '{unit_location}'" + ) + assert value == func_data[producer_key], ( + f"field '{producer_key}' did not round-trip: " + f"producer={func_data[producer_key]!r} unit={value!r}" + ) + + +def test_drift_prone_keys_stay_in_contract(): + """Self-check: the BUG-29 drift-prone key must remain in both contract maps + so this suite cannot silently go toothless if someone deletes the entry + (parity with the PHP schema test). If `is_inline` is dropped from a map, the + field-drift guard above evaporates -- fail loudly instead.""" + assert "is_inline" in FIELD_CONTRACT, "is_inline dropped from FIELD_CONTRACT -> guard disabled" + assert ANALYZER_CONTRACT.get("is_inline") == "isInline", "is_inline->isInline dropped from ANALYZER_CONTRACT -> guard disabled" diff --git a/libs/openant-core/tests/parsers/c/test_c_static_inline_header_resolution.py b/libs/openant-core/tests/parsers/c/test_c_static_inline_header_resolution.py new file mode 100644 index 00000000..279654b2 --- /dev/null +++ b/libs/openant-core/tests/parsers/c/test_c_static_inline_header_resolution.py @@ -0,0 +1,76 @@ +"""Bug 2 regression: a `static inline` function defined in an INCLUDED header +is callable from the including translation unit (the header-inline idiom — each +TU gets its own copy). The resolver dropped the edge because `_is_visible_from` +treated any cross-file `static` as internal-linkage-invisible, which is correct +for a `.c` translation unit but wrong for an included `.h` header. + +Paired must-preserve: tests/parsers/c/test_c_call_resolution_precision.py +::test_unique_name_fallback_skips_static_in_other_file — a `static` fn in another +`.c` must STILL NOT resolve cross-TU. This fix only un-blocks included headers. +""" +import sys +from pathlib import Path + +import pytest + +CORE = Path(__file__).resolve().parents[3] +if str(CORE) not in sys.path: + sys.path.insert(0, str(CORE)) + +pytest.importorskip("tree_sitter_c") + +from parsers.c.call_graph_builder import CallGraphBuilder # noqa: E402 + + +def _build(eo): + b = CallGraphBuilder(eo) + b.build_call_graph() + out = b.build() if hasattr(b, "build") else b.export() + return out.get("call_graph", {}) + + +def test_static_inline_in_included_header_resolves(): + eo = { + "functions": { + "m.c:caller": {"name": "caller", "file_path": "m.c", + "code": "int caller(int n){ return helper(n); }"}, + "h.h:helper": {"name": "helper", "file_path": "h.h", + "is_static": True, "is_inline": True, + "code": "static inline int helper(int x){ return x + 1; }"}, + }, + "includes": {"m.c": ["h.h"], "h.h": []}, + } + edges = _build(eo).get("m.c:caller", []) + assert "h.h:helper" in edges, f"static-inline header call dropped: {edges}" + + +def test_static_in_other_dot_c_still_rejected(): + # The #84 precision invariant, re-asserted locally: a static fn in another + # .c TU must NOT resolve (no #include relationship; genuinely file-local). + eo = { + "functions": { + "a.c:caller": {"name": "caller", "file_path": "a.c", + "code": "void caller(void){ helper(); }"}, + "b.c:helper": {"name": "helper", "file_path": "b.c", "is_static": True, + "code": "static void helper(void){}"}, + }, + "includes": {"a.c": [], "b.c": []}, + } + edges = _build(eo).get("a.c:caller", []) + assert "b.c:helper" not in edges, f"static helper() in b.c wrongly resolved cross-TU: {edges}" + + +def test_non_static_in_included_header_still_resolves(): + # Guard: a normal (extern) header function must keep resolving (no regression). + eo = { + "functions": { + "m.c:caller": {"name": "caller", "file_path": "m.c", + "code": "int caller(int n){ return ext_helper(n); }"}, + "api.h:ext_helper": {"name": "ext_helper", "file_path": "api.h", + "is_static": False, + "code": "int ext_helper(int x);"}, + }, + "includes": {"m.c": ["api.h"], "api.h": []}, + } + edges = _build(eo).get("m.c:caller", []) + assert "api.h:ext_helper" in edges, f"extern header call dropped: {edges}" diff --git a/libs/openant-core/tests/parsers/c/test_call_graph_builder_dispatch.py b/libs/openant-core/tests/parsers/c/test_call_graph_builder_dispatch.py new file mode 100644 index 00000000..1b6f645d --- /dev/null +++ b/libs/openant-core/tests/parsers/c/test_call_graph_builder_dispatch.py @@ -0,0 +1,396 @@ +"""Regression tests for the C/C++ call graph builder — member-dispatch (bug [51]). + +Bug report (C member-dispatch): + `Widget w; w.compute();` (and `Widget* w = ...; w->compute();`) should resolve + to `Widget::compute` (the method on w's known type), but the current resolver + discards the receiver and resolves `compute` base-name-first — linking a free + function `compute` (or, with two classes, the first-defined sibling method). + +Root cause: + parsers/c/call_graph_builder.py `_extract_call_name` returns ONLY the field + name for a `field_expression`, dropping the receiver. `_resolve_call` then has + no type context and resolves the bare base name (first/unique same-file match). + +Fix scope: SAME-FILE only. The receiver's static type is inferred from a local +variable declaration in the caller's body; if the type is unknown or has no such +method in the same translation unit, resolution FALLS BACK to the existing +base-name behavior (no false edge). + +Each test builds a real FunctionExtractor output from temp C++ source and runs the +real CallGraphBuilder pipeline, then asserts on the resulting edges. The RED cases +are constructed so the base-name-first resolver picks the WRONG target pre-fix. +""" + +import sys +import tempfile +from pathlib import Path + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +from parsers.c.call_graph_builder import CallGraphBuilder +from parsers.c.function_extractor import FunctionExtractor + + +def _extract(source: str, filename: str = "main.cpp") -> dict: + """Write source to a temp repo, run the real extractor, return its output.""" + tmpdir = tempfile.mkdtemp() + src_path = Path(tmpdir) / filename + src_path.write_text(source, encoding="utf-8") + extractor = FunctionExtractor(tmpdir) + return extractor.extract_all([filename]) + + +def _build(source: str, filename: str = "main.cpp") -> CallGraphBuilder: + builder = CallGraphBuilder(_extract(source, filename)) + builder.build_call_graph() + return builder + + +# --------------------------------------------------------------------------- # +# RED1: value-receiver member dispatch (Widget w; w.compute();) +# A free `compute` is present, so the base-name-first resolver mislinks to it +# pre-fix. Post-fix it must resolve to Widget::compute (the method on w's type). +# --------------------------------------------------------------------------- # + +_VALUE_SRC = """ +void compute() { } +class Widget { +public: + void compute() { } + void run() { + Widget w; + w.compute(); + } +}; +""" + + +def test_value_receiver_member_call_resolves_to_method(): + builder = _build(_VALUE_SRC) + caller = "main.cpp:Widget::run" + method = "main.cpp:Widget::compute" + free = "main.cpp:compute" + edges = builder.call_graph.get(caller, []) + assert method in edges, ( + f"Member call w.compute() did not resolve to the method on w's type.\n" + f" Expected edge to: {method}\n" + f" Got: {edges}\n" + f" Functions: {list(builder.functions)}" + ) + assert free not in edges, ( + f"Member call w.compute() wrongly linked to the free function compute.\n" + f" Got: {edges}" + ) + + +# --------------------------------------------------------------------------- # +# RED1b: pointer-receiver member dispatch (Widget* w = ...; w->compute();) +# --------------------------------------------------------------------------- # + +_PTR_SRC = """ +void compute() { } +class Widget { +public: + void compute() { } +}; +Widget* acquire(); +void run() { + Widget* w = acquire(); + w->compute(); +} +""" + + +def test_pointer_receiver_member_call_resolves_to_method(): + builder = _build(_PTR_SRC) + caller = "main.cpp:run" + method = "main.cpp:Widget::compute" + free = "main.cpp:compute" + edges = builder.call_graph.get(caller, []) + assert method in edges, ( + f"Member call w->compute() did not resolve to the method on w's type.\n" + f" Expected edge to: {method}\n" + f" Got: {edges}\n" + f" Functions: {list(builder.functions)}" + ) + assert free not in edges, ( + f"Member call w->compute() wrongly linked to the free function compute.\n" + f" Got: {edges}" + ) + + +# --------------------------------------------------------------------------- # +# PRECISION NEG1: unknown receiver type must fall back to base-name resolution +# (and never invent a type-dispatch edge). `x` is not declared in run(); the +# only same-file `compute` is a free function — fallback links it, as today. +# --------------------------------------------------------------------------- # + +_UNKNOWN_RECV_SRC = """ +void compute() { } +void run() { + x.compute(); +} +""" + + +def test_unknown_receiver_falls_back_to_base_name(): + builder = _build(_UNKNOWN_RECV_SRC) + caller = "main.cpp:run" + free = "main.cpp:compute" + assert builder.call_graph.get(caller, []) == [free], ( + f"Unknown-receiver member call did not resolve as the base-name fallback.\n" + f" Expected: [{free}]\n" + f" Got: {builder.call_graph.get(caller)}" + ) + + +# --------------------------------------------------------------------------- # +# PRECISION NEG2: plain free-function call (no receiver) unchanged +# --------------------------------------------------------------------------- # + +_FREE_SRC = """ +void helper() { } +void run() { + helper(); +} +""" + + +def test_plain_free_function_call_unchanged(): + builder = _build(_FREE_SRC) + caller = "main.cpp:run" + callee = "main.cpp:helper" + assert builder.call_graph.get(caller, []) == [callee], ( + f"Plain free-function call regressed.\n" + f" Expected: [{callee}]\n" + f" Got: {builder.call_graph.get(caller)}" + ) + + +# --------------------------------------------------------------------------- # +# PRECISION NEG3: two classes each with compute; B defined FIRST so the +# base-name-first resolver would pick B::compute pre-fix. `A a; a.compute()` +# must resolve to A::compute, NOT the sibling B::compute. +# --------------------------------------------------------------------------- # + +_TWO_CLASS_SRC = """ +class B { +public: + void compute() { } +}; +class A { +public: + void compute() { } +}; +void run() { + A a; + a.compute(); +} +""" + + +def test_member_call_resolves_to_correct_class_not_sibling(): + builder = _build(_TWO_CLASS_SRC) + caller = "main.cpp:run" + a_compute = "main.cpp:A::compute" + b_compute = "main.cpp:B::compute" + edges = builder.call_graph.get(caller, []) + assert a_compute in edges, ( + f"a.compute() did not resolve to A::compute.\n Got: {edges}" + ) + assert b_compute not in edges, ( + f"a.compute() wrongly linked to the sibling (first-defined) B::compute.\n" + f" Got: {edges}" + ) + + +# =========================================================================== # +# Bug [30]: virtual / inherited member dispatch — inheritance walk. +# +# `Base* b = ...; b->compute();` where compute is defined on Base (or an +# ancestor) and the receiver's STATICALLY-DECLARED type does NOT define it +# directly. Resolution must walk UP the base-class chain to the first +# ancestor that defines compute. This is the SOUND FLOOR: it resolves to the +# static type's method (or its nearest ancestor's), and deliberately does NOT +# link every derived override (a documented non-goal that creates false +# edges). Same-file only; no ancestor defining the method => no edge. +# =========================================================================== # + + +# --------------------------------------------------------------------------- # +# RED1 (inherited, non-virtual): Derived doesn't define compute -> walk up to +# Base::compute. A free `compute` is present so the pre-fix base-name-first +# resolver mislinks; pre-[30] the receiver-type path also fails (Derived has +# no compute) and falls back to the free function. +# --------------------------------------------------------------------------- # + +_INHERITED_SRC = """ +void compute() { } +struct Base { + void compute() { } +}; +struct Derived : Base { + void run() { + Derived* d = nullptr; + d->compute(); + } +}; +""" + + +def test_inherited_member_call_walks_up_to_base(): + builder = _build(_INHERITED_SRC) + caller = "main.cpp:Derived::run" + base_compute = "main.cpp:Base::compute" + free = "main.cpp:compute" + edges = builder.call_graph.get(caller, []) + assert base_compute in edges, ( + f"d->compute() did not walk up to the ancestor that defines compute.\n" + f" Expected edge to: {base_compute}\n" + f" Got: {edges}\n" + f" Functions: {list(builder.functions)}" + ) + assert free not in edges, ( + f"d->compute() wrongly linked to the free function compute.\n" + f" Got: {edges}" + ) + + +# --------------------------------------------------------------------------- # +# RED2 (virtual, static-type floor): Base declares+defines virtual compute; +# Derived overrides it. Called via a Base* receiver -> resolve to the STATIC +# type's method, Base::compute (the sound floor). Derived::compute is +# intentionally NOT also linked (documented non-goal: no over-approximation). +# --------------------------------------------------------------------------- # + +_VIRTUAL_SRC = """ +struct Base { + virtual void compute() { } +}; +struct Derived : Base { + void compute() override { } +}; +void run() { + Base* b = nullptr; + b->compute(); +} +""" + + +def test_virtual_member_call_resolves_to_static_type_floor(): + builder = _build(_VIRTUAL_SRC) + caller = "main.cpp:run" + base_compute = "main.cpp:Base::compute" + derived_compute = "main.cpp:Derived::compute" + edges = builder.call_graph.get(caller, []) + assert base_compute in edges, ( + f"b->compute() (Base* receiver) did not resolve to the static type's " + f"method Base::compute.\n Expected edge to: {base_compute}\n Got: {edges}" + ) + # Documented FLOOR: the derived override is NOT linked from a Base* call. + assert derived_compute not in edges, ( + f"b->compute() over-approximated: linked the derived override " + f"{derived_compute}. The [30] floor links only the static type's method.\n" + f" Got: {edges}" + ) + + +# --------------------------------------------------------------------------- # +# PRECISION NEG1: Derived DOES define compute (override) called via Derived* -> +# resolves to Derived::compute (its own), NOT Base::compute. The walk stops at +# the first definer (the receiver's own type). +# --------------------------------------------------------------------------- # + +_OVERRIDE_SRC = """ +struct Base { + virtual void compute() { } +}; +struct Derived : Base { + void compute() override { } + void run() { + Derived* d = nullptr; + d->compute(); + } +}; +""" + + +def test_override_resolves_to_own_method_not_ancestor(): + builder = _build(_OVERRIDE_SRC) + caller = "main.cpp:Derived::run" + derived_compute = "main.cpp:Derived::compute" + base_compute = "main.cpp:Base::compute" + edges = builder.call_graph.get(caller, []) + assert derived_compute in edges, ( + f"d->compute() (Derived* receiver, Derived overrides) did not resolve to " + f"its own Derived::compute.\n Got: {edges}" + ) + assert base_compute not in edges, ( + f"d->compute() walked past its own override to Base::compute. The walk " + f"must stop at the first definer.\n Got: {edges}" + ) + + +# --------------------------------------------------------------------------- # +# PRECISION NEG2: no ancestor defines the method -> NO type-dispatch edge, and +# no mislink to an unrelated free `compute`. Receiver type is known (Derived) +# but neither Derived nor its base Base defines `compute`; a free compute +# exists. The inheritance walk must return None (fall back). Fallback then +# links the free function (unchanged base-name behavior), but crucially the +# walk must NOT have invented a Base::compute / Derived::compute edge. +# --------------------------------------------------------------------------- # + +_NO_ANCESTOR_DEF_SRC = """ +void compute() { } +struct Base { + void other() { } +}; +struct Derived : Base { + void run() { + Derived* d = nullptr; + d->compute(); + } +}; +""" + + +def test_no_ancestor_defines_method_no_false_edge(): + builder = _build(_NO_ANCESTOR_DEF_SRC) + caller = "main.cpp:Derived::run" + edges = builder.call_graph.get(caller, []) + base_compute = "main.cpp:Base::compute" + derived_compute = "main.cpp:Derived::compute" + # The walk must not fabricate a method edge for a method no ancestor defines. + assert base_compute not in edges and derived_compute not in edges, ( + f"Inheritance walk fabricated a method edge for an undefined method.\n" + f" Got: {edges}" + ) + + +# --------------------------------------------------------------------------- # +# PRECISION NEG3: cycle in (malformed) inheritance -> the BFS terminates and +# does not crash. A: B, B: A, neither defines compute; a call via A* must not +# hang/recurse forever and must produce no fabricated method edge. +# --------------------------------------------------------------------------- # + +_CYCLE_SRC = """ +struct B; +struct A : B { + void run() { + A* a = nullptr; + a->compute(); + } +}; +struct B : A { +}; +""" + + +def test_inheritance_cycle_terminates_no_crash(): + # Must not raise / hang. + builder = _build(_CYCLE_SRC) + caller = "main.cpp:A::run" + edges = builder.call_graph.get(caller, []) + assert "main.cpp:A::compute" not in edges + assert "main.cpp:B::compute" not in edges diff --git a/libs/openant-core/tests/parsers/c/test_call_graph_builder_u1.py b/libs/openant-core/tests/parsers/c/test_call_graph_builder_u1.py new file mode 100644 index 00000000..5249fb5b --- /dev/null +++ b/libs/openant-core/tests/parsers/c/test_call_graph_builder_u1.py @@ -0,0 +1,178 @@ +"""Regression tests for the C call graph builder — builtin-name-collision leak. + +Bug report (BUG-NEW-2026-06-02-c-builtin_filter_leak, OpenAnt base 601e588): + A call to a *user-defined* function whose name collides with a C + stdlib/POSIX builtin (e.g. `close`, which is in STDLIB_FUNCTIONS) produces + NO edge in the call graph, because `_resolve_call` short-circuits with + `return None` on `_is_stdlib(call_name)` BEFORE it ever consults the + same-file user-function table. The callee is then falsely "isolated" / + unreachable. + +Root cause: + parsers/c/call_graph_builder.py `_resolve_call` — the `_is_stdlib` filter + runs first; the same-file lookup (step 1) is never reached for a colliding + name. + +Fix scope decision (see report): the pre-check is SCOPED to same-file +user-defined functions only. A genuine stdlib call (e.g. a real `printf` with +NO same-file definition) must still resolve to None — we must NOT route a +colliding name through a global cross-file single-match that could wrongly +link a real stdlib call to an unrelated same-named user function elsewhere. +""" + +import sys +from pathlib import Path + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +from parsers.c.call_graph_builder import CallGraphBuilder + + +def _make_extractor_output() -> dict: + """User function `close` (collides with POSIX stdlib) + a caller, same file. + + Mirrors function_extractor output shape: each function id is + `:` and func_data carries name / file_path / code. + """ + file_path = "main.c" + return { + "repository": "/tmp/fake", + "functions": { + f"{file_path}:close": { + "name": "close", + "file_path": file_path, + "code": "void close(int x) {\n}\n", + }, + f"{file_path}:caller": { + "name": "caller", + "file_path": file_path, + "code": "void caller(void) {\n close(3);\n}\n", + }, + }, + } + + +def test_user_function_colliding_with_stdlib_name_gets_edge(): + """caller() calls a same-file user `close` — the edge must exist.""" + builder = CallGraphBuilder(_make_extractor_output()) + builder.build_call_graph() + + caller = "main.c:caller" + callee = "main.c:close" + + assert callee in builder.call_graph[caller], ( + f"Forward call graph missing edge to same-file user function whose " + f"name collides with a stdlib builtin.\n" + f" Expected: {caller} -> {callee}\n" + f" Got: {builder.call_graph[caller]}" + ) + + +def test_user_function_colliding_with_stdlib_name_reverse_edge(): + """The colliding-name callee must list its caller in the reverse graph.""" + builder = CallGraphBuilder(_make_extractor_output()) + builder.build_call_graph() + + caller = "main.c:caller" + callee = "main.c:close" + + callers = builder.reverse_call_graph.get(callee, []) + assert caller in callers, ( + f"Reverse call graph missing caller for colliding-name user function.\n" + f" Expected to contain: {caller}\n" + f" Got: {callers}" + ) + + +def _make_real_stdlib_output() -> dict: + """A genuine stdlib call with NO same-file user definition. + + `printf` is a real stdlib call here and there is NO user-defined `printf` + anywhere — it must still resolve to nothing (no spurious edge). + """ + file_path = "real.c" + return { + "repository": "/tmp/fake", + "functions": { + f"{file_path}:greet": { + "name": "greet", + "file_path": file_path, + "code": 'void greet(void) {\n printf("hi");\n}\n', + }, + }, + } + + +def test_real_stdlib_call_still_filtered(): + """A real stdlib call with no same-file user def must produce NO edge.""" + builder = CallGraphBuilder(_make_real_stdlib_output()) + builder.build_call_graph() + + caller = "real.c:greet" + assert builder.call_graph[caller] == [], ( + f"Real stdlib call should not resolve to any edge.\n" + f" Got: {builder.call_graph[caller]}" + ) + + +def _make_cross_file_stdlib_output() -> dict: + """SCOPE guard: a real stdlib call in file A must NOT link to a same-named + user function defined in an UNRELATED file B (no include relationship). + + File A (`a.c`) calls `open(...)` — a genuine stdlib call. File B (`b.c`) + happens to define a user function `open`. With NO include linking them, + the call in A must NOT be wired to B's `open`. + """ + return { + "repository": "/tmp/fake", + "functions": { + "a.c:user_a": { + "name": "user_a", + "file_path": "a.c", + "code": "void user_a(void) {\n open(0);\n}\n", + }, + "b.c:open": { + "name": "open", + "file_path": "b.c", + "code": "void open(int x) {\n}\n", + }, + }, + } + + +def test_cross_file_stdlib_not_wrongly_linked(): + """A real stdlib call must not be linked to an unrelated same-named user + function in another (non-included) file.""" + builder = CallGraphBuilder(_make_cross_file_stdlib_output()) + builder.build_call_graph() + + caller = "a.c:user_a" + assert builder.call_graph[caller] == [], ( + f"Real stdlib call wrongly linked across files to an unrelated " + f"same-named user function.\n" + f" Got: {builder.call_graph[caller]}" + ) + + +def test_regex_fallback_resolves_colliding_user_function(): + """Sibling site: the regex fallback (_extract_calls_regex) must also resolve + a same-file user function whose name collides with a stdlib builtin, instead + of dropping it via an _is_stdlib pre-gate.""" + builder = CallGraphBuilder(_make_extractor_output()) + caller_id = "main.c:caller" + calls = builder._extract_calls_regex("close(3);", caller_id) + assert "main.c:close" in calls, ( + f"Regex fallback dropped the colliding-name same-file user function.\n" + f" Got: {calls}" + ) + + +def test_regex_fallback_still_filters_real_stdlib(): + """Sibling scope guard: the regex fallback must still drop a genuine stdlib + call with no same-file user definition.""" + builder = CallGraphBuilder(_make_real_stdlib_output()) + calls = builder._extract_calls_regex('printf("hi");', "real.c:greet") + assert calls == set(), ( + f"Regex fallback wrongly resolved a real stdlib call.\n Got: {calls}" + ) diff --git a/libs/openant-core/tests/parsers/c/test_function_extractor_u2.py b/libs/openant-core/tests/parsers/c/test_function_extractor_u2.py new file mode 100644 index 00000000..d92d9a99 --- /dev/null +++ b/libs/openant-core/tests/parsers/c/test_function_extractor_u2.py @@ -0,0 +1,150 @@ +"""Regression tests for the C/C++ FunctionExtractor — U2 blind-fix batch. + +Seven confirmed bugs (OpenAnt base 601e588 / 2e78d6a), all reproduced on the +real extractor (`FunctionExtractor(...).process_file(...)`): + + [15] out-of-line C++ constructor `Foo::Foo` recorded unit_type='method' + instead of 'constructor' (qualified name compared whole to class_name). + [14] out-of-line C++ destructor `Foo::~Foo` recorded 'method' not + 'destructor' (same qualified-vs-unqualified comparison bug). + [32] C++ struct member fn dropped class_name / unit_type='function' — the + tree walk only special-cased `class_specifier`, not `struct_specifier`. + [33] file-scope C++ lambda (`auto f = [](){...}`) never extracted — it lives + in a `declaration`/`init_declarator`/`lambda_expression`, not a + `function_definition`. + [35] C++ operator overload (`operator+`) never extracted — the + `operator_name` declarator node was unhandled, name resolved to None. + [39] explicit template specialization `g` collided with the primary + template `g` on func_id and silently overwrote it (template args + dropped from the id). + [40] free function inside a `namespace` wrongly carried class_name= + — namespace `::` qualifier conflated with a class qualifier. +""" + +import sys +import tempfile +from pathlib import Path + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +from parsers.c.function_extractor import FunctionExtractor + + +def _extract(filename: str, source: str) -> dict: + """Run the real extractor on a temp source file; return the functions dict.""" + repo = Path(tempfile.mkdtemp()).resolve() + fp = repo / filename + fp.write_text(source) + ex = FunctionExtractor(str(repo)) + ex.process_file(fp) + return ex.functions + + +def _find(functions: dict, predicate): + for fid, data in functions.items(): + if predicate(fid, data): + return data + return None + + +# ---------------------------------------------------------------- [15] ctor +def test_outofline_constructor_classified_constructor(): + functions = _extract("ctor.cpp", "Foo::Foo() { }\n") + data = _find(functions, lambda fid, d: d["name"] == "Foo::Foo") + assert data is not None, f"ctor not extracted; got {list(functions)}" + assert data["unit_type"] == "constructor", ( + f"expected constructor, got {data['unit_type']!r}" + ) + + +# ---------------------------------------------------------------- [14] dtor +def test_outofline_destructor_classified_destructor(): + functions = _extract("dtor.cpp", "Foo::~Foo() { }\n") + data = _find(functions, lambda fid, d: d["name"] == "Foo::~Foo") + assert data is not None, f"dtor not extracted; got {list(functions)}" + assert data["unit_type"] == "destructor", ( + f"expected destructor, got {data['unit_type']!r}" + ) + + +# ------------------------------------------------------------ [32] struct member +def test_struct_member_method_metadata(): + functions = _extract("m.cpp", "struct Point {\n int dist() { return 0; }\n};\n") + data = _find(functions, lambda fid, d: d["name"].endswith("dist")) + assert data is not None, f"struct member not extracted; got {list(functions)}" + assert data["unit_type"] == "method", ( + f"expected method, got {data['unit_type']!r}" + ) + assert data["class_name"] == "Point", ( + f"expected class_name Point, got {data['class_name']!r}" + ) + + +# ------------------------------------------------------------ [32b] union member +def test_union_member_method_metadata(): + functions = _extract("u.cpp", "union U {\n int tag() { return 0; }\n};\n") + data = _find(functions, lambda fid, d: d["name"].endswith("tag")) + assert data is not None, f"union member not extracted; got {list(functions)}" + assert data["unit_type"] == "method", ( + f"expected method, got {data['unit_type']!r}" + ) + assert data["class_name"] == "U", ( + f"expected class_name U, got {data['class_name']!r}" + ) + + +# ---------------------------------------------------------------- [33] lambda +def test_file_scope_lambda_extracted(): + functions = _extract( + "m.cpp", "int ctrl() { return 0; }\nauto f = [](int x){ return x + 1; };\n" + ) + data = _find(functions, lambda fid, d: d["name"] == "f") + assert data is not None, f"lambda 'f' not extracted; got {list(functions)}" + + +# ---------------------------------------------------------------- [35] operator +def test_operator_overload_extracted(): + functions = _extract( + "m.cpp", + "int ctrl() { return 0; }\n\nclass V {\npublic:\n" + " int operator+(int x) { return x + 1; }\n};\n", + ) + data = _find(functions, lambda fid, d: "operator" in d["name"]) + assert data is not None, f"operator+ not extracted; got {list(functions)}" + assert data["class_name"] == "V", ( + f"expected class_name V, got {data['class_name']!r}" + ) + + +# ------------------------------------------------------------ [39] template spec +def test_template_specialization_distinct_from_primary(): + functions = _extract( + "m.cpp", + "int control(){return 1;}\n" + "template T g(T x){return x;}\n" + "template<> int g(int x){return x+1;}\n", + ) + # Both the primary template `g` and the specialization `g` must survive. + spec = _find(functions, lambda fid, d: "g" in fid or "g" in d["name"]) + primary = _find( + functions, + lambda fid, d: (fid.endswith(":g") or d["name"] == "g"), + ) + assert spec is not None, f"g specialization absent; got {list(functions)}" + assert primary is not None, f"primary g absent; got {list(functions)}" + + +# ------------------------------------------------------------ [40] namespace free fn +def test_namespace_free_function_no_class_name(): + functions = _extract( + "m.cpp", "namespace ns {\nint freefunc(int x) {\n return x;\n}\n}\n" + ) + data = _find(functions, lambda fid, d: d["name"].endswith("freefunc")) + assert data is not None, f"freefunc not extracted; got {list(functions)}" + assert data["class_name"] is None, ( + f"expected class_name None for namespace free fn, got {data['class_name']!r}" + ) + assert data["unit_type"] != "method", ( + f"namespace free fn must not be a method, got {data['unit_type']!r}" + ) diff --git a/libs/openant-core/tests/parsers/c/test_unit_generator_u3.py b/libs/openant-core/tests/parsers/c/test_unit_generator_u3.py new file mode 100644 index 00000000..f7456e25 --- /dev/null +++ b/libs/openant-core/tests/parsers/c/test_unit_generator_u3.py @@ -0,0 +1,72 @@ +"""Regression test for BUG 29 (c schema_field_drift in unit_generator.py). + +Bug report (OpenAnt blind mining, 2026-06-04): + The C function extractor computes and stores `is_inline` for every function + (function_extractor._process_function_node, func_data['is_inline']), but the + unit generator's create_unit() never copies it into the assembled unit's + `metadata` block. Sibling boolean flags `is_static` and `is_exported` ARE + carried there, so `is_inline` is silently dropped at unit assembly. + +Reproduction: + Source `m.c` = 'inline int add(int a, int b) {\\n return a + b;\\n}\\n' + target = m.c:add, check = metadata, expected is_inline == True. + +This test drives the REAL extractor to produce func_data, feeds a call-graph- +shaped dict to the REAL UnitGenerator, and asserts the unit exposes is_inline +at the same location as its sibling flags (unit['metadata']). +""" + +import sys +from pathlib import Path + +import pytest + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +from parsers.c.function_extractor import FunctionExtractor +from parsers.c.unit_generator import UnitGenerator + + +INLINE_SRC = "inline int add(int a, int b) {\n return a + b;\n}\n" + + +def _extract_functions(tmp_path: Path) -> dict: + """Run the real C function extractor on an inline function and return func_data map.""" + src_file = tmp_path / "m.c" + src_file.write_text(INLINE_SRC) + + extractor = FunctionExtractor(str(tmp_path)) + result = extractor.extract_all(files=["m.c"]) + return result["functions"] + + +def _call_graph_data(functions: dict, repo: Path) -> dict: + """Shape the extractor output as call-graph data that UnitGenerator consumes.""" + return { + "repository": str(repo), + "functions": functions, + "call_graph": {}, + "reverse_call_graph": {}, + } + + +def test_extractor_produces_is_inline(tmp_path): + """Sanity: the producer actually emits is_inline=True for an inline function.""" + functions = _extract_functions(tmp_path) + assert "m.c:add" in functions, f"add not extracted; got {list(functions)}" + assert functions["m.c:add"]["is_inline"] is True + + +def test_unit_exposes_is_inline(tmp_path): + """BUG 29: the assembled unit must carry is_inline alongside its sibling flags.""" + functions = _extract_functions(tmp_path) + gen = UnitGenerator(_call_graph_data(functions, tmp_path)) + unit = gen.create_unit("m.c:add", functions["m.c:add"]) + + # Sibling flags live in unit['metadata']; is_inline must too. + assert "is_inline" in unit["metadata"], ( + "is_inline dropped at unit assembly; " + f"metadata keys = {sorted(unit['metadata'])}" + ) + assert unit["metadata"]["is_inline"] is True diff --git a/libs/openant-core/tests/parsers/go/test_go_schema_completeness.py b/libs/openant-core/tests/parsers/go/test_go_schema_completeness.py new file mode 100644 index 00000000..d764e6ad --- /dev/null +++ b/libs/openant-core/tests/parsers/go/test_go_schema_completeness.py @@ -0,0 +1,188 @@ +"""Schema-completeness contract test for the Go parser (BUG-NEW 5 family guard). + +BUG-NEW 5 is a *cross-parser schema drift*. The Go parser is a separate Go +binary whose `FunctionInfo` records (`parsers/go/go_parser/types.go`) use +**camelCase** json keys (`unitType`, `startLine`, `endLine`, `isExported`, +`filePath`, `className`). Every other parser (python/ruby/php/c/zig) emits +**snake_case** (`unit_type`, `start_line`, ...). The Python reachability / +entry-point consumers read snake_case -- e.g. +`utilities/agentic_enhancer/entry_point_detector.py` reads +`func_data.get('unit_type')`. So for Go records read out of `call_graph.json` +they got `None`, and any unit_type-based logic (entry-point classification, +statistics, the module_level check) was silently broken for Go. + +The fix normalizes the Go function records to **snake_case** at the single +Python ingestion boundary that builds `call_graph.json`'s `functions` map +(`parsers/go/test_pipeline.py`), matching the consumer contract and every +other parser. No Go rebuild; the analyzer_output.json camelCase contract +(consumed by the camelCase-aware analyzer surface) is intentionally left alone. + +Design: + * REACH_SRC is a Go program with a silent `func main()` that calls a helper + (no decorators, no input patterns) -- the ONLY thing that makes `main` an + entry point is its unit_type/name. This exercises BUG-4 (+name:main) and + BUG-5 (unit_type readable) together. + * The real Go binary + real test_pipeline.py produce call_graph.json and the + reachability-filtered dataset.json. + * Tests assert (1) the ingested function records expose snake_case + unit_type/file_path/start_line (not None), and (2) main seeds reachability + with a unit_type-derived entry-point reason and the helper is reachable. + +If the Go toolchain / subprocess is unavailable the end-to-end tests skip and a +normalization unit-test on a representative camelCase record still runs. +""" + +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +_GO_PARSER_DIR = _CORE_ROOT / "parsers" / "go" / "go_parser" +_TEST_PIPELINE = _CORE_ROOT / "parsers" / "go" / "test_pipeline.py" + +# A silent main() calling a helper: nothing but unit_type/name makes main an +# entry point (no decorators, no request.* / argv input patterns). +REACH_SRC = """package main + +import "fmt" + +func helper(x int) int { +\treturn x * 2 +} + +func main() { +\tfmt.Println(helper(21)) +} +""" + +GO_MOD = "module bug5repo\n\ngo 1.21\n" + +# Snake-case keys the Python reachability/entry-point consumers read out of +# call_graph.json's `functions` records. These are None today for Go records. +CONSUMER_SNAKE_KEYS = ["unit_type", "file_path", "start_line", "end_line", "is_exported"] + + +def _go_available(): + return shutil.which("go") is not None or (_GO_PARSER_DIR / "go_parser").exists() + + +@pytest.fixture(scope="module") +def go_pipeline_output(tmp_path_factory): + """Run the real Go binary + real test_pipeline.py; return (call_graph, dataset).""" + if not _go_available(): + pytest.skip("Go toolchain / go_parser binary unavailable") + + repo = tmp_path_factory.mktemp("bug5_repo") + (repo / "main.go").write_text(REACH_SRC) + (repo / "go.mod").write_text(GO_MOD) + out = tmp_path_factory.mktemp("bug5_out") + + cmd = [ + sys.executable, str(_TEST_PIPELINE), str(repo), + "--output", str(out), + "--processing-level", "reachable", + "--skip-tests", + ] + proc = subprocess.run( + cmd, cwd=str(_CORE_ROOT), + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, timeout=300, + ) + cg_path = out / "call_graph.json" + ds_path = out / "dataset.json" + if proc.returncode != 0 or not cg_path.exists() or not ds_path.exists(): + pytest.skip(f"Go pipeline did not produce outputs (env-flaky):\n{proc.stdout[-2000:]}") + + return json.loads(cg_path.read_text()), json.loads(ds_path.read_text()) + + +@pytest.mark.parametrize("snake_key", CONSUMER_SNAKE_KEYS) +def test_go_function_record_exposes_snake_case(go_pipeline_output, snake_key): + """Each ingested Go function record must expose the snake_case key the + Python consumer reads (RED: only camelCase present, so snake key is None).""" + call_graph, _dataset = go_pipeline_output + functions = call_graph.get("functions", {}) + assert functions, "Go pipeline produced no function records" + for func_id, fd in functions.items(): + assert snake_key in fd, ( + f"Go function {func_id!r} missing snake_case key {snake_key!r}; " + f"consumer would read None. keys = {sorted(fd)}" + ) + + +def test_go_main_record_unit_type_is_main(go_pipeline_output): + """The main() record's snake_case unit_type must read 'main' (RED: None).""" + call_graph, _dataset = go_pipeline_output + functions = call_graph.get("functions", {}) + main_recs = [fd for fd in functions.values() if fd.get("name") == "main"] + assert main_recs, f"no main record; ids = {list(functions)}" + assert main_recs[0].get("unit_type") == "main", ( + f"main unit_type not readable as snake_case; got {main_recs[0].get('unit_type')!r} " + f"(camel unitType = {main_recs[0].get('unitType')!r})" + ) + + +def test_go_main_seeds_reachability_via_unit_type(go_pipeline_output): + """End-to-end payoff: a silent func main() calling helper() seeds reachability + with a unit_type-derived entry-point reason, and the helper is reachable.""" + _call_graph, dataset = go_pipeline_output + units = {u["id"]: u for u in dataset.get("units", [])} + main_id = next((i for i in units if i.endswith(":main")), None) + helper_id = next((i for i in units if i.endswith(":helper")), None) + assert main_id and helper_id, f"missing main/helper units; ids = {list(units)}" + + assert units[main_id].get("is_entry_point") is True, "main not seeded as entry point" + # The unit_type-derived reason must be present now that unit_type is readable. + reason = units[main_id].get("entry_point_reason", "") + assert "unit_type:main" in reason, ( + f"main entry-point reason lacks unit_type:main (BUG-5 still drifting); reason = {reason!r}" + ) + assert units[helper_id].get("reachable") is True, "helper not reachable from main" + + +def test_normalize_camel_record_to_snake_full_schema(): + """BUG-5 re-verify (no Go toolchain needed): a representative camelCase Go + FunctionInfo record normalizes to the FULL snake_case consumer schema, + including the parameters / returns / is_async fields that were previously + omitted from normalize_go_function_records.""" + from parsers.go.test_pipeline import normalize_go_function_records + camel = {"f.go:Pkg.M": { + "name": "M", "code": "func ...", "startLine": 10, "endLine": 20, + "unitType": "method", "className": "Pkg", "isExported": True, + "package": "main", "filePath": "f.go", "receiver": "Pkg", + "parameters": ["x int"], "returns": ["error"], "isAsync": True, + "decorators": ["// note"], + }} + out = normalize_go_function_records(camel)["f.go:Pkg.M"] + expected = { + "name": "M", "unit_type": "method", "file_path": "f.go", + "start_line": 10, "end_line": 20, "is_exported": True, + "class_name": "Pkg", "package": "main", "receiver": "Pkg", + "parameters": ["x int"], "returns": ["error"], "is_async": True, + "decorators": ["// note"], + } + for k, v in expected.items(): + assert out.get(k) == v, f"{k!r}: got {out.get(k)!r}, expected {v!r}" + # No camelCase keys leak through. + assert not any(k in out for k in ("unitType", "filePath", "isAsync", "className")), out + + +def test_normalize_is_idempotent_on_snake_records(): + """Already-snake records pass through unchanged (idempotency).""" + from parsers.go.test_pipeline import normalize_go_function_records + snake = {"f.go:h": { + "name": "h", "unit_type": "function", "code": "", "file_path": "f.go", + "start_line": 1, "end_line": 2, "package": "main", "receiver": "", + "is_exported": False, "class_name": "", "decorators": [], + "parameters": [], "returns": [], "is_async": False, + }} + once = normalize_go_function_records(snake) + twice = normalize_go_function_records(once) + assert once == twice, "normalization not idempotent on snake records" + assert once["f.go:h"]["unit_type"] == "function" diff --git a/libs/openant-core/tests/parsers/javascript/test_block_scoped_functions.py b/libs/openant-core/tests/parsers/javascript/test_block_scoped_functions.py new file mode 100644 index 00000000..ee19b87c --- /dev/null +++ b/libs/openant-core/tests/parsers/javascript/test_block_scoped_functions.py @@ -0,0 +1,141 @@ +"""Bug: block-scoped function declarations are silently dropped from BOTH the +function inventory and the call graph. + +The analyzer enumerated functions via `sourceFile.getFunctions()` (top-level +only) in extractFunctionsFromFile AND buildCallGraphForFile, so a function +declared inside any block (if/else, try/catch, for/while/switch, bare {}) was +invisible — never a unit, never a reachability node, never an entry point. For a +SAST tool a vulnerable function gated behind `if (process.env.X)` or in a +`catch` fallback is then never analyzed. + +Investigated independent + expert + judge. Settled scope (verified against the +`node` runtime): + * MODULE-LEVEL block functions (no function-like ancestor) MUST be surfaced. + * Functions nested inside another function/method stay omitted (their text + rides inside the parent unit). + * Functions inside a `static {}` class block stay omitted (uncallable). + * Both loops patched in lockstep so callGraph keys == functions keys (the + backstop must not mask a missing-edge gap). + * Sibling-block same-name functions are both runtime-reachable -> keep BOTH + via a `#L` suffix (collision-only; unique names keep the clean id). +""" +import json +import shutil +import subprocess +from pathlib import Path + +import pytest + +PARSERS_JS_DIR = Path(__file__).parent.parent.parent.parent / "parsers" / "javascript" +NODE_MODULES = PARSERS_JS_DIR / "node_modules" + +pytestmark = pytest.mark.skipif( + not shutil.which("node") or not NODE_MODULES.exists(), + reason="Node.js or JS parser npm dependencies not available", +) + + +def _analyze(tmp_path, source, filename="a.js"): + repo = tmp_path / "r" + repo.mkdir(exist_ok=True) + fp = repo / filename + fp.write_text(source) + cmd = ["node", str(PARSERS_JS_DIR / "typescript_analyzer.js"), str(repo), str(fp)] + res = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + assert res.returncode == 0, f"analyzer failed:\n{res.stderr}" + return json.loads(res.stdout) + + +def _names(out): + return sorted(k.split(":", 1)[1] for k in out["functions"]) + + +# --- the core bug: module-level block functions surfaced in BOTH maps -------- + +@pytest.mark.parametrize("wrap", [ + "if (c) { %s }", + "if (c) {} else { %s }", + "try { %s } catch (e) {}", + "try {} catch (e) { %s }", + "for (let i=0;i<1;i++) { %s }", + "while (c) { %s }", + "switch (x) { case 1: { %s } }", + "{ %s }", +]) +def test_block_scoped_function_is_extracted(tmp_path, wrap): + src = "function topLevel(){ return 1; }\n" + wrap % "function vuln(){ return sink(); }" + out = _analyze(tmp_path, src) + assert "vuln" in _names(out), f"block function dropped from inventory: {_names(out)}" + # and it must be a real call-graph node, not a backstop-filled empty entry + vuln_id = next(k for k in out["functions"] if k.endswith(":vuln")) + assert vuln_id in out["callGraph"], "vuln missing from callGraph" + edges = out["callGraph"][vuln_id] + # callGraph edges are dicts: {"name": "sink", "resolved": bool, ...} + edge_names = [e.get("name") if isinstance(e, dict) else e for e in edges] + assert "sink" in edge_names, ( + f"vuln's outgoing edge to sink lost (backstop masked it): {edges}" + ) + + +def test_incoming_call_to_block_function_resolves(tmp_path): + # A call TO a module-level block-scoped function must resolve in the RESOLVED + # call graph, so the block function is reachable (not pruned). Guards the + # sibling resolver site (_buildResolvedGraphs) — without it the function is a + # unit but has no resolved incoming edge. + src = "function caller(){ return helper(); }\nif (c) { function helper(){ return 1; } }\n" + out = _analyze(tmp_path, src) + caller_id = next(k for k in out["functions"] if k.endswith(":caller")) + helper_id = next(k for k in out["functions"] if k.endswith(":helper")) + assert helper_id in out["call_graph"].get(caller_id, []), ( + f"caller->helper not resolved: {out['call_graph'].get(caller_id)}" + ) + assert caller_id in out["reverse_call_graph"].get(helper_id, []), ( + f"block fn has no resolved incoming edge: {out['reverse_call_graph'].get(helper_id)}" + ) + + +def test_invariant_callgraph_keys_equal_functions_keys(tmp_path): + src = "if (c) { function a(){ b(); } }\nfunction b(){}\n" + out = _analyze(tmp_path, src) + assert set(out["functions"]) == set(out["callGraph"]), ( + "callGraph keys diverge from functions keys (lockstep broken)" + ) + + +# --- keep-both for sibling-block same-name (both runtime-reachable) ----------- + +def test_sibling_block_same_name_keeps_both(tmp_path): + src = ( + "if (c) { function dup(){ return ifBranch(); } }\n" + "else { function dup(){ return elseBranch(); } }\n" + ) + out = _analyze(tmp_path, src) + dups = [k for k in out["functions"] if k.split(":", 1)[1].startswith("dup")] + assert len(dups) == 2, f"both sibling-block dup defs must survive; got {dups}" + bodies = " ".join(out["functions"][k]["code"] for k in dups) + assert "ifBranch" in bodies and "elseBranch" in bodies, f"a branch was dropped: {bodies}" + + +# --- the omit cases (must NOT over-extract) ----------------------------------- + +def test_function_nested_in_function_stays_omitted(tmp_path): + out = _analyze(tmp_path, "function outer(){ function inner(){ return 1; } return inner(); }") + assert "inner" not in _names(out), f"nested-in-function should be omitted: {_names(out)}" + assert "outer" in _names(out) + + +def test_function_in_static_block_stays_omitted(tmp_path): + # class static-init block: the function is block-scoped to the initializer + # and callable nowhere -> must not become a (false-positive) unit. + out = _analyze(tmp_path, "class C { static { function s(){ return 1; } } }") + assert "s" not in _names(out), f"static-block function should be omitted: {_names(out)}" + + +# --- regression: unique top-level id unchanged -------------------------------- + +def test_unique_top_level_id_unchanged(tmp_path): + out = _analyze(tmp_path, "function solo(){ return 1; }") + ids = [k for k in out["functions"] if k.endswith("solo")] + assert len(ids) == 1 and ids[0].endswith(":solo") and "#L" not in ids[0], ( + f"unique-name id must stay byte-identical: {ids}" + ) diff --git a/libs/openant-core/tests/parsers/php/test_call_graph_builder_u5.py b/libs/openant-core/tests/parsers/php/test_call_graph_builder_u5.py new file mode 100644 index 00000000..0da7342c --- /dev/null +++ b/libs/openant-core/tests/parsers/php/test_call_graph_builder_u5.py @@ -0,0 +1,166 @@ +"""Regression tests for two confirmed PHP CallGraphBuilder edge-resolution bugs. + +Both drive the FULL real pipeline (FunctionExtractor -> CallGraphBuilder) on +real PHP source and assert that the expected call-graph *edge* is present, +matching the way the bugs were reproduced on the live parser. + +[BUG 20] variable-function dataflow loss + ``$f = 'helper'; $f();`` -- the call node is a ``function_call_expression`` + whose callee child is a ``variable_name`` (``$f``), not a ``name`` / + ``qualified_name``. ``_resolve_function_call`` only matched ``name`` / + ``identifier`` / ``qualified_name`` callees, so ``func_name`` stayed None + and the call was dropped -- the edge caller -> helper was never emitted. + Fix: when the callee is a ``variable_name``, follow a single unconditional + string-literal binding (``$f = 'helper';``) in the caller's body and resolve + the bound name. + +[BUG 52] parent:: scoped-call dispatch + ``parent::greet()`` -- the scope child is a ``relative_scope`` node (text + ``parent``), not a ``name`` / ``qualified_name``. ``_resolve_scoped_call`` + only populated ``scope`` from ``name`` / ``qualified_name``, so both + ``scope`` and ``method_name`` stayed None and the call was dropped. The + pre-existing ``parent`` branch was unreachable dead code. ``parent`` + additionally cannot be resolved by the same-class self-call resolver: the + method lives on the PARENT class, possibly in another file. Fix: read the + ``relative_scope``; for ``parent`` look up the caller class's ``superclass`` + (class->parent index) and resolve the method on that parent class via the + cross-file class-method resolver. +""" + +import sys +import tempfile +from pathlib import Path + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +from parsers.php.function_extractor import FunctionExtractor +from parsers.php.call_graph_builder import CallGraphBuilder + + +def _build_graph(files: dict) -> dict: + """Run the real PHP pipeline on {filename: source}; return the call_graph.""" + repo = tempfile.mkdtemp() + for name, src in files.items(): + Path(repo, name).write_text(src) + + extractor = FunctionExtractor(repo) + extractor_output = extractor.extract_all(list(files.keys())) + + builder = CallGraphBuilder(extractor_output) + builder.build_call_graph() + return builder.call_graph + + +# --- BUG 20: variable-function dataflow ------------------------------------ + +_VARFN_SOURCE = ( + " helper edge.""" + graph = _build_graph({"main.php": _VARFN_SOURCE}) + edges = graph.get("main.php:caller", []) + assert "main.php:helper" in edges, ( + "variable-function edge dropped: $f='helper';$f() lost the edge to " + "helper (callee is a variable_name; string binding not followed).\n" + f" caller edges: {edges!r}" + ) + + +def test_variable_function_conditional_binding_not_followed(): + """Precision guard: a non-unique / reassigned binding must NOT be resolved.""" + src = ( + "parent cross-file lookup).\n" + f" Child.run edges: {edges!r}" + ) + + +def test_parent_scoped_call_same_file_emits_edge(): + """`parent::greet()` must also resolve when parent is in the same file.""" + src = ( + " dict: + repo = tempfile.mkdtemp() + with open(os.path.join(repo, filename), "w") as fh: + fh.write(php_source) + return FunctionExtractor(repo).extract_all([filename]) + + +def test_anon_class_method_attributed_to_synthetic_class(): + src = ( + " dict: + """Run the real extractor on a source string; return the full export dict.""" + repo = tempfile.mkdtemp() + Path(repo, filename).write_text(php_source) + extractor = FunctionExtractor(repo) + return extractor.extract_all([filename]) + + +# --- BUG 7: nested function never extracted ------------------------------- + +_NESTED_SOURCE = ( + "` disambiguation the Python extractor already +uses), not prefer-first/larger. Collision-only: a unique name keeps its clean id. +""" +import sys +import tempfile +from pathlib import Path + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +from parsers.php.function_extractor import FunctionExtractor + + +def _extract(php_source: str, filename: str = "cond.php") -> dict: + repo = tempfile.mkdtemp() + Path(repo, filename).write_text(php_source) + return FunctionExtractor(repo).extract_all([filename]) + + +def _named(out: dict, name: str): + return [fid for fid, d in out["functions"].items() if d.get("name") == name] + + +def test_conditional_ifelse_keeps_both_branches(): + src = ( + " how to read the mapped value off the assembled unit. +# Each entry is (extractor_func_data_key, unit_location_reader). Extend this map +# whenever the extractor grows an analysis-relevant field that the unit exposes. +_CONTRACT = { + "name": lambda u: u["code"]["primary_origin"]["function_name"], + "class_name": lambda u: u["code"]["primary_origin"]["class_name"], + "unit_type": lambda u: u["unit_type"], + "start_line": lambda u: u["code"]["primary_origin"]["start_line"], + "end_line": lambda u: u["code"]["primary_origin"]["end_line"], + "parameters": lambda u: u["metadata"]["parameters"], + # The two field-drift legs this guard exists for: + "namespace_name": lambda u: u["metadata"]["namespace"], # BUG 43 + "is_static": lambda u: u["metadata"]["is_static"], # BUG 28 +} + + +def test_extractor_to_unit_field_contract(): + """Every contracted producer field must reach its mapped unit location.""" + functions, units = _run_pipeline(_SOURCE) + assert functions, "extractor produced no functions" + + failures = [] + for func_id, func_data in functions.items(): + unit = units.get(func_id) + assert unit is not None, f"no unit generated for {func_id}" + for producer_key, reader in _CONTRACT.items(): + produced = func_data.get(producer_key) + exposed = reader(unit) + if exposed != produced: + failures.append( + f"{func_id}: key {producer_key!r} produced={produced!r} " + f"but unit exposes {exposed!r}" + ) + + assert not failures, "Field-contract drift detected:\n " + "\n ".join(failures) + + +def test_visibility_is_a_known_producer_gap(): + """KNOWN PRODUCER GAP (out of scope here, recorded as a regression marker). + + The extractor computes ``_get_visibility`` but ``_process_function_node`` + never stores a ``visibility`` key in ``func_data``; the unit_generator then + fabricates a default ``'public'`` via ``func_data.get('visibility', + 'public')``. So a ``private`` method is reported as ``public``. This is a + PRODUCER omission (function_extractor.py), distinct from the two CONSUMER + field-drift bugs fixed in this run -- left for a separate fix. When the + extractor starts emitting ``visibility``, this test will fail, prompting a + move of ``visibility`` into the ``_CONTRACT`` map above. + """ + functions, units = _run_pipeline(_SOURCE) + # The private method's true visibility is 'private', but the producer never + # emits the key, so the unit currently mis-reports it as the default. + priv = units["m.php:C.p"] + assert "visibility" not in functions["m.php:C.p"], ( + "Extractor now emits 'visibility' -- promote it into _CONTRACT and " + "drop this gap marker." + ) + assert priv["metadata"]["visibility"] == "public", ( + "Unit still defaults visibility; gap unchanged." + ) + + +def test_static_and_namespace_present_in_contract(): + """Self-check: the two bug-driven keys are actually in the contract map. + + Guards against someone silently deleting the drift-prone entries and + leaving a green-but-toothless test. + """ + assert "is_static" in _CONTRACT + assert "namespace_name" in _CONTRACT diff --git a/libs/openant-core/tests/parsers/php/test_unit_generator_u7.py b/libs/openant-core/tests/parsers/php/test_unit_generator_u7.py new file mode 100644 index 00000000..50445e40 --- /dev/null +++ b/libs/openant-core/tests/parsers/php/test_unit_generator_u7.py @@ -0,0 +1,120 @@ +"""Regression tests for the PHP unit generator's metadata field-contract. + +Two confirmed producer->consumer field-drift bugs between +``parsers/php/function_extractor.py`` (PRODUCER, writes ``func_data``) and +``parsers/php/unit_generator.py`` (CONSUMER, ``UnitGenerator.create_unit``): + +[BUG 28] ``is_static`` + The extractor computes ``func_data["is_static"]`` (function_extractor.py, + ``_process_function_node``), but ``create_unit`` never copies it into the + assembled unit's ``metadata``. The unit cannot tell static methods apart. + +[BUG 43] ``namespace`` key-drift + The extractor writes the declared namespace under the key + ``func_data["namespace_name"]``, but ``create_unit`` reads + ``func_data.get("namespace")`` -- a key-name mismatch -- so the unit's + ``metadata["namespace"]`` is ALWAYS ``None``. + +These tests drive the FULL real pipeline (FunctionExtractor -> +CallGraphBuilder -> UnitGenerator) on real PHP source, matching the way the +bugs were reproduced on the live parser. + +Construct-isolation note (BUG 43): + A *non-braced* ``namespace App\\Foo;`` declaration is, in the tree-sitter + grammar, a sibling of the following ``class_declaration`` rather than its + parent, and the extractor's traversal does NOT propagate the namespace to + those siblings -- a SEPARATE extractor traversal bug that also yields a + ``None`` namespace. To test the key-drift in ``create_unit`` in isolation + (and avoid confounding it with that traversal bug), these tests use a + *braced* ``namespace App\\Foo { ... }`` block, for which the extractor + demonstrably produces a correct ``namespace_name``. See the module-level + note in the report for the separate non-braced finding. +""" + +import sys +import tempfile +from pathlib import Path + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +from parsers.php.function_extractor import FunctionExtractor +from parsers.php.call_graph_builder import CallGraphBuilder +from parsers.php.unit_generator import UnitGenerator + + +def _generate_units(php_source: str, filename: str = "m.php") -> dict: + """Run the real PHP pipeline on a source string; return units keyed by id.""" + repo = tempfile.mkdtemp() + Path(repo, filename).write_text(php_source) + + extractor = FunctionExtractor(repo) + extractor_output = extractor.extract_all([filename]) + + builder = CallGraphBuilder(extractor_output) + builder.build_call_graph() + + result = UnitGenerator(builder.export()).generate_units() + return {u["id"]: u for u in result["units"]} + + +# --- BUG 43: namespace key-drift ------------------------------------------ + +_NS_SOURCE = ( + "`) machinery, and closes +the `__module__` leak by covering every def/class span. +""" +import sys +import tempfile +from pathlib import Path + +import pytest + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +from parsers.python.function_extractor import FunctionExtractor + + +def _extract(src: str) -> dict: + repo = Path(tempfile.mkdtemp()).resolve() + (repo / "m.py").write_text(src) + ex = FunctionExtractor(str(repo)) + ex.process_file(repo / "m.py") + return ex.functions + + +def _names(functions: dict): + return sorted(k.split(":", 1)[1] for k in functions) + + +@pytest.mark.parametrize("wrap", [ + "if X:\n {d}", + "if X:\n pass\nelse:\n {d}", + "try:\n {d}\nexcept Exception:\n pass", + "try:\n pass\nexcept Exception:\n {d}", + "try:\n pass\nfinally:\n {d}", + "for i in r:\n {d}", + "while X:\n {d}", + "with open('x') as f:\n {d}", +]) +def test_block_scoped_def_is_extracted(wrap): + src = "def top(): pass\n" + wrap.format(d="def blk(): return sink()") + assert "blk" in _names(_extract(src)), f"block def dropped: {_names(_extract(src))}" + + +def test_match_case_def_is_extracted(): + src = "def top(): pass\nmatch v:\n case 1:\n def handler(): return 1\n" + assert "handler" in _names(_extract(src)) + + +def test_async_and_decorated_block_defs_extracted(): + src = ( + "import functools\n" + "if X:\n" + " async def afn(): return 1\n" + "if Y:\n" + " @functools.cache\n" + " def dfn(): return 2\n" + ) + names = _names(_extract(src)) + assert "afn" in names and "dfn" in names, names + + +def test_class_in_block_and_its_methods_extracted(): + # A class inside a block, and its methods, must surface. The method + # `Hidden.m` in the functions inventory proves the block-nested class was + # descended into and processed (the class itself lands in `ex.classes`). + src = "if TYPE_CHECKING:\n class Hidden:\n def m(self): return 1\n" + names = _names(_extract(src)) + assert any(n.endswith("Hidden.m") for n in names), names + + +def test_function_internal_block_def_extracted_no_duplicate(): + src = ( + "def outer():\n" + " def direct(): return 1\n" + " if c:\n" + " def blocked(): return 2\n" + ) + names = _names(_extract(src)) + assert names.count("direct") == 1, f"direct duplicated: {names}" + assert "blocked" in names, f"function-internal block def dropped: {names}" + + +def test_sibling_block_same_name_keeps_both(): + src = ( + "if c:\n def view(): return a()\n" + "else:\n def view(): return b()\n" + ) + views = [n for n in _names(_extract(src)) if n.startswith("view")] + assert len(views) == 2, f"both if/else view defs must survive: {views}" + + +def test_block_def_colliding_with_top_level_keeps_both(): + src = "def dup(): return 1\nif c:\n def dup(): return 2\n" + dups = [n for n in _names(_extract(src)) if n.startswith("dup")] + assert len(dups) == 2, f"block def must not clobber top-level same-name: {dups}" + + +def test_module_unit_does_not_leak_block_def_body(): + # The block def's body (incl. its sink) must move into its own unit, not + # leak verbatim into the synthetic :__module__ text. + src = "if X:\n def hidden(req):\n return __import__('os').system(req)\n" + fns = _extract(src) + assert "hidden" in _names(fns), "hidden not surfaced" + mod = next((v for k, v in fns.items() if k.endswith(":__module__")), None) + if mod is not None: + assert "system(req)" not in mod.get("code", ""), ( + "block def body leaked into __module__" + ) diff --git a/libs/openant-core/tests/parsers/python/test_call_graph_builder_u8.py b/libs/openant-core/tests/parsers/python/test_call_graph_builder_u8.py new file mode 100644 index 00000000..50528df4 --- /dev/null +++ b/libs/openant-core/tests/parsers/python/test_call_graph_builder_u8.py @@ -0,0 +1,155 @@ +"""Regression tests for four confirmed call_graph_builder bugs (u8 bundle). + +Each test drives the REAL parser pipeline: write source file(s) to a temp +repo, run FunctionExtractor, build the call graph with CallGraphBuilder, then +assert the edge that should exist (or the false edge that must NOT exist). + +Bugs covered: + [10] builtin-filter leak — user fn named like a stdlib module (`time`) loses + its call edge because _is_builtin short-circuits before same-file lookup. + [12] dataflow alias — `fn = helper; fn()` loses the edge to `helper`. + [27] import over-resolution — `import alpha; alpha.run()` spuriously resolves + to a free function named `alpha` (matched on module tail, not call name). + [46] inherited-self dispatch — `self.shared()` calling an inherited base + method isn't resolved. +""" + +import sys +import tempfile +from pathlib import Path + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +from parsers.python.function_extractor import FunctionExtractor +from parsers.python.call_graph_builder import CallGraphBuilder + + +def _build(files: dict) -> CallGraphBuilder: + """Write {relpath: source} into a temp repo, run the real pipeline.""" + d = tempfile.mkdtemp() + for rel, src in files.items(): + p = Path(d) / rel + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(src) + extractor_output = FunctionExtractor(d).extract_all() + builder = CallGraphBuilder(extractor_output) + builder.build_call_graph() + return builder + + +# ---------------------------------------------------------------- [10] +def test_user_fn_named_like_stdlib_module_resolves(): + """A user fn named `time` (a stdlib module name) must still get its edge.""" + builder = _build({ + "m.py": "def time():\n return 1\n\ndef main():\n time()\n", + }) + caller = "m.py:main" + callee = "m.py:time" + assert callee in builder.call_graph[caller], ( + f"Edge {caller} -> {callee} missing (builtin filter leaked).\n" + f" Got: {builder.call_graph[caller]}" + ) + + +def test_genuine_builtin_call_not_linked_to_unrelated_samename_fn(): + """SCOPE guard: a genuine `time()` builtin call in file b.py must NOT link + to a user `def time()` in an UNRELATED file a.py (no cross-file leak).""" + builder = _build({ + "a.py": "def time():\n return 1\n", + "b.py": "def consume():\n return time()\n", + }) + caller = "b.py:consume" + bad = "a.py:time" + assert bad not in builder.call_graph.get(caller, []), ( + f"False cross-file edge {caller} -> {bad} (builtin call wrongly linked).\n" + f" Got: {builder.call_graph.get(caller, [])}" + ) + + +# ---------------------------------------------------------------- [12] +def test_function_value_alias_resolves(): + """`fn = helper; fn()` must produce main -> helper.""" + builder = _build({ + "m.py": "def helper():\n return 1\n\ndef main():\n fn = helper\n fn()\n", + }) + caller = "m.py:main" + callee = "m.py:helper" + assert callee in builder.call_graph[caller], ( + f"Edge {caller} -> {callee} missing (alias not followed).\n" + f" Got: {builder.call_graph[caller]}" + ) + + +# ---------------------------------------------------- [12] single-assign GUARD +def test_alias_reassignment_not_resolved(): + """GUARD (reassignment): `fn = a; fn = b; fn()` is last-write-wins, so the + binding is a "maybe". The guard must NOT assert a definite edge to EITHER + target — pinned behavior: no alias edge at all (fall through to no edge).""" + builder = _build({ + "m.py": ( + "def a():\n return 1\n\n" + "def b():\n return 2\n\n" + "def main():\n fn = a\n fn = b\n fn()\n" + ), + }) + caller = "m.py:main" + edges = builder.call_graph.get(caller, []) + assert "m.py:a" not in edges and "m.py:b" not in edges, ( + f"reassigned alias asserted a maybe-binding as definite: {edges}" + ) + + +def test_alias_conditional_binding_not_resolved(): + """GUARD (conditional): an alias bound inside if/else is not unconditional, + so it must NOT be resolved (no edge to either branch's target).""" + builder = _build({ + "m.py": ( + "def a():\n return 1\n\n" + "def b():\n return 2\n\n" + "def main(cond):\n" + " if cond:\n fn = a\n else:\n fn = b\n" + " fn()\n" + ), + }) + caller = "m.py:main" + edges = builder.call_graph.get(caller, []) + assert "m.py:a" not in edges and "m.py:b" not in edges, ( + f"conditional alias resolved despite non-unconditional binding: {edges}" + ) + + +# ---------------------------------------------------------------- [27] +def test_import_module_call_does_not_false_link_to_samename_free_fn(): + """`import alpha; alpha.run()` must NOT link caller -> free fn `alpha`.""" + builder = _build({ + "m.py": "import alpha\n\ndef alpha():\n return 1\n\ndef caller():\n return alpha.run()\n", + }) + caller = "m.py:caller" + bad = "m.py:alpha" + assert bad not in builder.call_graph.get(caller, []), ( + f"False edge {caller} -> {bad} (matched module tail, not call name).\n" + f" Got: {builder.call_graph.get(caller, [])}" + ) + + +# ---------------------------------------------------------------- [46] +def test_inherited_self_method_resolves(): + """self.shared() inherited from a base class must resolve to Base.shared.""" + builder = _build({ + "m.py": ( + "class Base:\n" + " def shared(self):\n" + " return 1\n" + "\n" + "class Child(Base):\n" + " def run(self):\n" + " return self.shared()\n" + ), + }) + caller = "m.py:Child.run" + callee = "m.py:Base.shared" + assert callee in builder.call_graph[caller], ( + f"Edge {caller} -> {callee} missing (inherited self-call not resolved).\n" + f" Got: {builder.call_graph[caller]}" + ) diff --git a/libs/openant-core/tests/parsers/python/test_call_graph_import_resolution.py b/libs/openant-core/tests/parsers/python/test_call_graph_import_resolution.py new file mode 100644 index 00000000..3e6cde0d --- /dev/null +++ b/libs/openant-core/tests/parsers/python/test_call_graph_import_resolution.py @@ -0,0 +1,134 @@ +"""Import-resolution precision/recall for the Python call graph builder. + +Bug (byproduct-deepcheck 2026-06-12, P4): `_resolve_import` Strategy 2 matched a +function's file to the imported module with a bare string `endswith` on the dotted +module path: + + module_path.endswith(expected_module) or expected_module.endswith(module_path) + +That `endswith` matched ACROSS component boundaries: `from utils import helper` +bound to `extra_utils.py:helper` because `'extra_utils'.endswith('utils')`. No +Python module layout produces that (the module `utils` is not the file +`extra_utils.py`), so it is a genuinely-false edge. + +Fix: match by dotted COMPONENTS, in EITHER direction — the imported module is a +component-suffix of the file path (a repo-root prefix on the file, `src/pkg/auth.py` +for `pkg.auth`) OR the file path is a component-suffix of the imported module (the +repo-IS-the-package self-import, `auth.py` == `myapp/auth.py` recorded shallow, for +`from myapp.auth import ...`). Both directions are real layouts and are kept; only the +substring-crossing matches are dropped. + +Note: `from pkg.auth import login` -> a top-level `auth.py` is NOT provably false — +it is structurally identical to the repo-is-package self-import, so under the +"never miss a real edge" rule it is kept (a false edge is triageable; a dropped real +edge silently removes attack surface from the reachability filter). + +These tests pin both the precision drops (substring-crossing edges gone) AND recall +(every component-suffix match, both directions, still resolves). +""" + +import sys +from pathlib import Path + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +from parsers.python.call_graph_builder import CallGraphBuilder + + +def _build(functions: dict, imports: dict) -> CallGraphBuilder: + b = CallGraphBuilder({"repository": "/tmp/fake", "imports": imports, + "classes": {}, "functions": functions}) + b.build_call_graph() + return b + + +def _fn(file_path, name, code, class_name=None): + return {"name": name, "qualified_name": (f"{class_name}.{name}" if class_name else name), + "file_path": file_path, "class_name": class_name, "unit_type": "function", "code": code} + + +# ---------- PRECISION: substring-crossing (genuinely-false) edges must NOT appear ---------- + +def test_no_false_edge_substring_module(): + """`from utils import helper` must NOT bind to extra_utils.py (substring suffix).""" + fns = {"main.py:run": _fn("main.py", "run", "def run():\n return helper()\n"), + "utils.py:other": _fn("utils.py", "other", "def other():\n return 0\n"), + "extra_utils.py:helper": _fn("extra_utils.py", "helper", "def helper():\n return 1\n")} + b = _build(fns, {"main.py": {"helper": "utils.helper"}}) + assert "extra_utils.py:helper" not in b.call_graph.get("main.py:run", []), ( + f"false edge to extra_utils.py:helper: {b.call_graph.get('main.py:run')}") + + +def test_no_false_edge_substring_module_auth(): + """`from auth import login` must NOT bind to extra_auth.py.""" + fns = {"main.py:run": _fn("main.py", "run", "def run():\n return login()\n"), + "extra_auth.py:login": _fn("extra_auth.py", "login", "def login():\n return 1\n")} + b = _build(fns, {"main.py": {"login": "auth.login"}}) + assert "extra_auth.py:login" not in b.call_graph.get("main.py:run", []), ( + f"false edge to extra_auth.py:login: {b.call_graph.get('main.py:run')}") + + +# ---------- RECALL: legitimate imports must STILL resolve (no findings lost) ---------- + +def test_recall_exact_package_path(): + """`from pkg.auth import login` with a real pkg/auth.py MUST resolve.""" + fns = {"main.py:run": _fn("main.py", "run", "def run():\n return login()\n"), + "pkg/auth.py:login": _fn("pkg/auth.py", "login", "def login():\n return 1\n")} + b = _build(fns, {"main.py": {"login": "pkg.auth.login"}}) + assert "pkg/auth.py:login" in b.call_graph.get("main.py:run", []), ( + f"legit package import dropped: {b.call_graph.get('main.py:run')}") + + +def test_recall_repo_root_prefix(): + """`from pkg.auth import login` with src/pkg/auth.py (repo-root prefix) MUST resolve.""" + fns = {"main.py:run": _fn("main.py", "run", "def run():\n return login()\n"), + "src/pkg/auth.py:login": _fn("src/pkg/auth.py", "login", "def login():\n return 1\n")} + b = _build(fns, {"main.py": {"login": "pkg.auth.login"}}) + assert "src/pkg/auth.py:login" in b.call_graph.get("main.py:run", []), ( + f"legit repo-root-prefixed import dropped: {b.call_graph.get('main.py:run')}") + + +def test_recall_top_level_module(): + """`from auth import login` with a real top-level auth.py MUST resolve.""" + fns = {"main.py:run": _fn("main.py", "run", "def run():\n return login()\n"), + "auth.py:login": _fn("auth.py", "login", "def login():\n return 1\n")} + b = _build(fns, {"main.py": {"login": "auth.login"}}) + assert "auth.py:login" in b.call_graph.get("main.py:run", []), ( + f"legit top-level import dropped: {b.call_graph.get('main.py:run')}") + + +def test_recall_repo_is_package_self_import(tmp_path=None): + """`from myapp.auth import login` with the repo recorded shallow (`auth.py` == + `myapp/auth.py`) MUST resolve — the repo-IS-the-package self-import. This is the + reverse-direction (import deeper than the recorded file) case; a forward-only + component match would WRONGLY drop it (recall regression caught by review).""" + fns = {"main.py:run": _fn("main.py", "run", "def run():\n return login()\n"), + "auth.py:login": _fn("auth.py", "login", "def login():\n return 1\n")} + b = _build(fns, {"main.py": {"login": "myapp.auth.login"}}) + assert "auth.py:login" in b.call_graph.get("main.py:run", []), ( + f"repo-is-package self-import dropped (reverse-direction recall regression): " + f"{b.call_graph.get('main.py:run')}") + + +def test_recall_reverse_prefix_nested(tmp_path=None): + """`from a.b.c import func` with the file recorded as `b/c.py` MUST resolve + (nested namespace/package layout — file shallower than the absolute import).""" + fns = {"main.py:run": _fn("main.py", "run", "def run():\n return func()\n"), + "b/c.py:func": _fn("b/c.py", "func", "def func():\n return 1\n")} + b = _build(fns, {"main.py": {"func": "a.b.c.func"}}) + assert "b/c.py:func" in b.call_graph.get("main.py:run", []), ( + f"nested reverse-prefix import dropped: {b.call_graph.get('main.py:run')}") + + +def test_recall_relative_import_preserved(): + """Adversarial recall guard: a relative import `from . import helper` (stored as a + bare name with no module) must STILL resolve — the precision fix must not drop the + name-only resolution path the old code provided for relative imports.""" + fns = {"pkg/main.py:run": _fn("pkg/main.py", "run", "def run():\n return helper()\n"), + "pkg/util.py:helper": _fn("pkg/util.py", "helper", "def helper():\n return 1\n")} + # `from . import helper` -> extractor stores imports['helper'] = 'helper' (bare, no module) + b = _build(fns, {"pkg/main.py": {"helper": "helper"}}) + assert "pkg/util.py:helper" in b.call_graph.get("pkg/main.py:run", []), ( + f"relative-import resolution dropped by the precision fix: " + f"{b.call_graph.get('pkg/main.py:run')}") diff --git a/libs/openant-core/tests/parsers/python/test_call_graph_self_calls.py b/libs/openant-core/tests/parsers/python/test_call_graph_self_calls.py index a92c23cc..8b98c0a9 100644 --- a/libs/openant-core/tests/parsers/python/test_call_graph_self_calls.py +++ b/libs/openant-core/tests/parsers/python/test_call_graph_self_calls.py @@ -138,3 +138,65 @@ def test_callee_is_not_isolated_in_statistics(): f" call_graph: {builder.call_graph}\n" f" reverse_call_graph: {builder.reverse_call_graph}" ) + + +# --- self / cls alias edges (byproduct-deepcheck 2026-06-12) --- +# `obj = self; obj.method()` and `alias = cls; alias.method()` are real +# self/class-method calls, but _resolve_call_node only matched a receiver +# literally named `self`/`cls`, so the edge was DROPPED (under-resolution -> +# the callee can look unreachable). Fixing ADDS the missing edge; it never +# removes one (raises reachability, never lowers it). + +def _alias_extractor_output(body: str, decorator: str = "") -> dict: + """Two methods on one class; `caller` reaches `target` via a self/cls alias.""" + file_path = "m.py" + dec = f" {decorator}\n" if decorator else "" + return { + "repository": "/tmp/fake", + "imports": {file_path: {}}, + "classes": {f"{file_path}:C": {"name": "C", "file_path": file_path}}, + "functions": { + f"{file_path}:C.target": { + "name": "target", "qualified_name": "C.target", "file_path": file_path, + "class_name": "C", "unit_type": "method", + "code": f"{dec} def target(self):\n return 1\n", + }, + f"{file_path}:C.caller": { + "name": "caller", "qualified_name": "C.caller", "file_path": file_path, + "class_name": "C", "unit_type": "method", "code": body, + }, + }, + } + + +def test_self_alias_method_call_edge(): + """`obj = self; obj.target()` must produce the C.caller -> C.target edge.""" + body = " def caller(self):\n obj = self\n return obj.target()\n" + builder = CallGraphBuilder(_alias_extractor_output(body)) + builder.build_call_graph() + assert "m.py:C.target" in builder.call_graph["m.py:C.caller"], ( + f"self-alias edge missing; got {builder.call_graph['m.py:C.caller']}" + ) + + +def test_cls_alias_method_call_edge(): + """`alias = cls; alias.target()` in a classmethod must produce the edge.""" + body = (" @classmethod\n def caller(cls):\n" + " alias = cls\n return alias.target()\n") + builder = CallGraphBuilder(_alias_extractor_output(body, decorator="@classmethod")) + builder.build_call_graph() + assert "m.py:C.target" in builder.call_graph["m.py:C.caller"], ( + f"cls-alias edge missing; got {builder.call_graph['m.py:C.caller']}" + ) + + +def test_reassigned_alias_does_not_force_self_edge(): + """Guard: a name reassigned away from self must NOT be treated as a self-alias + (single-unconditional binding only) — keeps the fix sound, no spurious edges.""" + body = (" def caller(self, other):\n obj = self\n" + " obj = other\n return obj.target()\n") + builder = CallGraphBuilder(_alias_extractor_output(body)) + builder.build_call_graph() + assert "m.py:C.target" not in builder.call_graph.get("m.py:C.caller", []), ( + "reassigned alias wrongly resolved to a self-method edge" + ) diff --git a/libs/openant-core/tests/parsers/python/test_callgraph_symmetry.py b/libs/openant-core/tests/parsers/python/test_callgraph_symmetry.py new file mode 100644 index 00000000..e382ca27 --- /dev/null +++ b/libs/openant-core/tests/parsers/python/test_callgraph_symmetry.py @@ -0,0 +1,70 @@ +"""Canonical per-parser invariant: every call-graph node is a real function. + +`set(call_graph.keys()) ⊆ set(functions.keys())` and the same for +`reverse_call_graph` — no call-graph key may reference a function id that the +inventory doesn't contain. The fixture exercises top-level, nested, method, and +block-scoped (if/try/for/with) defs so the invariant is checked across every +emit path (a block def must appear in BOTH maps, not just one). +""" +import sys +import tempfile +from pathlib import Path + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +from parsers.python.function_extractor import FunctionExtractor +from parsers.python.call_graph_builder import CallGraphBuilder + +_FIXTURE = ( + "def top():\n" + " return helper()\n" + "def helper():\n" + " return 1\n" + "class C:\n" + " def m(self):\n" + " return self.helper2()\n" + " def helper2(self):\n" + " return 2\n" + "if FLAG:\n" + " def block_fn():\n" + " return top()\n" + "try:\n" + " def fallback():\n" + " return 3\n" + "except Exception:\n" + " pass\n" +) + + +def _build(): + d = tempfile.mkdtemp() + (Path(d) / "m.py").write_text(_FIXTURE) + builder = CallGraphBuilder(FunctionExtractor(d).extract_all()) + builder.build_call_graph() + return builder + + +def test_callgraph_keys_subset_of_functions(): + b = _build() + fns = set(b.functions) + extra = set(b.call_graph) - fns + assert not extra, f"call_graph references non-inventory ids: {sorted(extra)}" + + +def test_reverse_callgraph_keys_subset_of_functions(): + b = _build() + fns = set(b.functions) + extra = set(b.reverse_call_graph) - fns + assert not extra, f"reverse_call_graph references non-inventory ids: {sorted(extra)}" + + +def test_block_scoped_def_is_a_callgraph_node_with_its_edge(): + # The block-scoped def must be in the inventory AND carry its real edge, + # never an orphan / backstop-empty entry. + b = _build() + block_id = next(k for k in b.functions if k.endswith(":block_fn")) + top_id = next(k for k in b.functions if k.endswith(":top")) + assert top_id in b.call_graph.get(block_id, []), ( + f"block_fn -> top edge missing: {b.call_graph.get(block_id)}" + ) diff --git a/libs/openant-core/tests/parsers/python/test_function_extractor_u9.py b/libs/openant-core/tests/parsers/python/test_function_extractor_u9.py new file mode 100644 index 00000000..4206a050 --- /dev/null +++ b/libs/openant-core/tests/parsers/python/test_function_extractor_u9.py @@ -0,0 +1,231 @@ +"""Regression tests for FunctionExtractor (u9 blind bug batch, 2026-06-08). + +Six confirmed bugs in parsers/python/function_extractor.py: + + BUG 9 nested-def inside a function body is never extracted as a unit. + BUG 21 module-level `handler = lambda ...` is never extracted as a unit. + BUG 23 decorated function start_line points at `def` line, not the decorator + (off-by-one; off-by-N for stacked decorators) while `code` includes them. + BUG 34 `@x.setter` is classified 'method' not 'property', AND getter/setter + share a func_id so the setter silently overwrites the getter. + BUG 45 a method of a class nested inside another class is never extracted. + BUG 48 unit_type='test' for any file whose path merely CONTAINS the substring + 'test' (e.g. latest.py), instead of a path-component check. + +Each test writes a minimal source file under a tmp dir, runs the extractor, +and asserts on the returned dict (the parser operates on files, so we model +the real entry path: FunctionExtractor(repo).extract_all([rel_path])). +""" + +import sys +from pathlib import Path + +import pytest + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +from parsers.python.function_extractor import FunctionExtractor + + +def _extract(tmp_path: Path, filename: str, source: str) -> dict: + """Write `source` to tmp_path/filename, extract, return the full result dict.""" + f = tmp_path / filename + f.parent.mkdir(parents=True, exist_ok=True) + f.write_text(source, encoding="utf-8") + extractor = FunctionExtractor(str(tmp_path)) + return extractor.extract_all([filename]) + + +# --- BUG 9: nested def inside a function body --------------------------------- + +def test_bug9_nested_def_is_extracted(tmp_path): + source = "def outer():\n def inner():\n return 1\n return inner\n" + result = _extract(tmp_path, "m.py", source) + func_names = {fd["name"] for fd in result["functions"].values()} + assert "inner" in func_names, ( + f"nested def 'inner' not extracted. functions={list(result['functions'])}" + ) + + +# --- BUG 21: module-level lambda assigned to a name --------------------------- + +def test_bug21_module_level_lambda_is_extracted(tmp_path): + source = "handler = lambda req: req.upper()\n" + result = _extract(tmp_path, "m.py", source) + func_names = {fd["name"] for fd in result["functions"].values()} + assert "handler" in func_names, ( + f"module-level lambda 'handler' not extracted. functions={list(result['functions'])}" + ) + + +# --- BUG 23: decorated start_line includes the decorator line ----------------- + +def test_bug23_decorated_start_line_includes_decorator(tmp_path): + source = "class Foo:\n @staticmethod\n def bar():\n pass\n" + result = _extract(tmp_path, "m.py", source) + fid = "m.py:Foo.bar" + assert fid in result["functions"], f"missing {fid}: {list(result['functions'])}" + # @staticmethod is on line 2; def bar is on line 3. start_line must be 2. + assert result["functions"][fid]["start_line"] == 2, ( + f"decorated start_line off-by-one: got {result['functions'][fid]['start_line']}, expected 2" + ) + + +def test_bug23_stacked_decorators_start_line(tmp_path): + source = ( + "def d1(f):\n return f\n\n" + "def d2(f):\n return f\n\n" + "@d1\n@d2\ndef target():\n pass\n" + ) + result = _extract(tmp_path, "m.py", source) + fid = "m.py:target" + assert fid in result["functions"], f"missing {fid}: {list(result['functions'])}" + # @d1 on line 7, @d2 on line 8, def target on line 9. start_line must be 7. + assert result["functions"][fid]["start_line"] == 7, ( + f"stacked-decorator start_line wrong: got {result['functions'][fid]['start_line']}, expected 7" + ) + + +# --- BUG 34: property getter/setter ------------------------------------------ + +def test_bug34_property_setter_classified_and_not_collapsed(tmp_path): + source = ( + "class C:\n" + " @property\n" + " def x(self):\n" + " return self._x\n" + "\n" + " @x.setter\n" + " def x(self, value):\n" + " self._x = value\n" + ) + result = _extract(tmp_path, "m.py", source) + funcs = result["functions"] + # Getter keeps the CANONICAL id; the setter is disambiguated by ROLE in the + # qualified_name (order-independent) -- so both survive AND func_id stays + # path:qualified_name (the call_graph_builder reconstruction invariant). + assert "m.py:C.x" in funcs, f"getter lost its canonical id; ids={list(funcs)}" + assert "m.py:C.x.setter" in funcs, f"setter not stored under role id; ids={list(funcs)}" + getter, setter = funcs["m.py:C.x"], funcs["m.py:C.x.setter"] + # Invariant: func_id == path:qualified_name for BOTH units. + assert getter["qualified_name"] == "C.x", getter["qualified_name"] + assert setter["qualified_name"] == "C.x.setter", setter["qualified_name"] + # Both classified 'property'; roles distinguished. + assert getter["unit_type"] == "property" and setter["unit_type"] == "property" + assert getter["property_role"] == "getter", getter["property_role"] + assert setter["property_role"] == "setter", setter["property_role"] + + +# --- BUG 45: nested class members --------------------------------------------- + +def test_bug45_nested_class_method_is_extracted(tmp_path): + source = ( + "class Outer:\n" + " class Inner:\n" + " def deep(self):\n" + " return 1\n" + " def shallow(self):\n" + " return 2\n" + ) + result = _extract(tmp_path, "m.py", source) + deep_units = [fd for fd in result["functions"].values() if fd["name"] == "deep"] + assert deep_units, ( + f"nested-class method 'deep' not extracted. functions={list(result['functions'])}" + ) + + +# --- BUG 48: test classification by substring vs path component --------------- + +def test_bug48_substring_test_not_classified_as_test(tmp_path): + source = "def compute():\n return 1\n" + result = _extract(tmp_path, "latest.py", source) + fid = "latest.py:compute" + assert fid in result["functions"], f"missing {fid}: {list(result['functions'])}" + assert result["functions"][fid]["unit_type"] == "function", ( + f"'latest.py' misclassified as test: got {result['functions'][fid]['unit_type']}, expected 'function'" + ) + + +def test_bug48_real_test_file_still_classified_test(tmp_path): + """Guard: a genuine test file path component must STILL classify as test.""" + source = "def helper():\n return 1\n" + result = _extract(tmp_path, "tests/test_thing.py", source) + fid = "tests/test_thing.py:helper" + assert fid in result["functions"], f"missing {fid}: {list(result['functions'])}" + assert result["functions"][fid]["unit_type"] == "test", ( + f"genuine test file not classified test: got {result['functions'][fid]['unit_type']}" + ) + + +# --- BUG 34 re-verification (2026-06-09): property-classification edge cases ----- +def test_bug34_cached_property_getter_classified_property(tmp_path): + source = ( + "from functools import cached_property\n" + "class C:\n" + " @cached_property\n" + " def x(self):\n" + " return self._x\n" + ) + funcs = _extract(tmp_path, "m.py", source)["functions"] + assert "m.py:C.x" in funcs, list(funcs) + assert funcs["m.py:C.x"]["property_role"] == "getter" + assert funcs["m.py:C.x"]["unit_type"] == "property", ( + f"cached_property getter misclassified: {funcs['m.py:C.x']['unit_type']}" + ) + + +def test_bug34_functools_cached_property_classified_property(tmp_path): + source = ( + "import functools\n" + "class C:\n" + " @functools.cached_property\n" + " def x(self):\n" + " return self._x\n" + ) + funcs = _extract(tmp_path, "m.py", source)["functions"] + assert funcs["m.py:C.x"]["unit_type"] == "property" + + +def test_bug34_orphan_setter_not_lost(tmp_path): + source = "class C:\n @x.setter\n def x(self, v):\n self._x = v\n" + funcs = _extract(tmp_path, "m.py", source)["functions"] + assert "m.py:C.x.setter" in funcs, list(funcs) + assert funcs["m.py:C.x.setter"]["property_role"] == "setter" + assert funcs["m.py:C.x.setter"]["unit_type"] == "property" + + +def test_bug34_isolated_deleter(tmp_path): + source = "class C:\n @x.deleter\n def x(self):\n del self._x\n" + funcs = _extract(tmp_path, "m.py", source)["functions"] + assert "m.py:C.x.deleter" in funcs, list(funcs) + assert funcs["m.py:C.x.deleter"]["property_role"] == "deleter" + + +def test_bug34_two_classes_same_property_no_collision(tmp_path): + source = ( + "class C:\n @property\n def x(self):\n return 1\n" + "class D:\n @property\n def x(self):\n return 2\n" + ) + funcs = _extract(tmp_path, "m.py", source)["functions"] + assert "m.py:C.x" in funcs and "m.py:D.x" in funcs, list(funcs) + + +def test_bug34_non_property_decorator_not_misclassified(tmp_path): + """Narrowing guard: a method whose decorator merely CONTAINS 'property' as + a substring (not the property protocol) must NOT classify as 'property'.""" + source = ( + "class C:\n" + " @app.property_route\n" + " def a(self):\n" + " return 1\n" + " @some_property_validator\n" + " def b(self):\n" + " return 2\n" + ) + funcs = _extract(tmp_path, "m.py", source)["functions"] + a = next(fd for fd in funcs.values() if fd["name"] == "a") + b = next(fd for fd in funcs.values() if fd["name"] == "b") + assert a["unit_type"] != "property", f"@app.property_route mis-classified: {a['unit_type']}" + assert b["unit_type"] != "property", f"@some_property_validator mis-classified: {b['unit_type']}" + assert a["property_role"] is None and b["property_role"] is None diff --git a/libs/openant-core/tests/parsers/python/test_python_schema_completeness.py b/libs/openant-core/tests/parsers/python/test_python_schema_completeness.py new file mode 100644 index 00000000..58e44a74 --- /dev/null +++ b/libs/openant-core/tests/parsers/python/test_python_schema_completeness.py @@ -0,0 +1,71 @@ +"""Canonical per-parser schema completeness: every emitted function unit carries +the schema-contract fields downstream consumers (call graph, unit generator, +entry-point detector, dataset) read. Run across top-level, nested, method, +async, decorated, and block-scoped defs so no emit path drops a field. +""" +import sys +import tempfile +from pathlib import Path + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +from parsers.python.function_extractor import FunctionExtractor + +# Fields every function unit must expose (present-as-key; value may be None/[]). +_REQUIRED_FIELDS = { + "name", "qualified_name", "file_path", "start_line", "end_line", + "code", "parameters", "unit_type", "is_async", "decorators", + "class_name", +} + +_FIXTURE = ( + "import functools\n" + "def top(a, b):\n" + " return a + b\n" + "async def atop():\n" + " return 1\n" + "@functools.cache\n" + "def decorated():\n" + " return 2\n" + "class C:\n" + " def method(self):\n" + " def nested():\n" + " return 3\n" + " return nested()\n" + "if FLAG:\n" + " def block_fn(x):\n" + " return x\n" + " async def block_async():\n" + " return 4\n" +) + + +def _functions(): + repo = Path(tempfile.mkdtemp()).resolve() + (repo / "m.py").write_text(_FIXTURE) + ex = FunctionExtractor(str(repo)) + ex.process_file(repo / "m.py") + return {k: v for k, v in ex.functions.items() if not k.endswith(":__module__")} + + +def test_every_function_has_required_schema_fields(): + fns = _functions() + assert fns, "fixture produced no functions" + for fid, data in fns.items(): + missing = _REQUIRED_FIELDS - set(data) + assert not missing, f"{fid} missing schema fields: {sorted(missing)}" + + +def test_block_scoped_defs_present_and_well_formed(): + names = {k.split(":", 1)[1] for k in _functions()} + for expected in ("block_fn", "block_async"): + assert expected in names, f"{expected} not surfaced: {sorted(names)}" + + +def test_field_value_types(): + for fid, data in _functions().items(): + assert isinstance(data["name"], str) and data["name"], fid + assert isinstance(data["parameters"], list), fid + assert isinstance(data["start_line"], int), fid + assert isinstance(data["is_async"], bool), fid diff --git a/libs/openant-core/tests/parsers/ruby/test_call_graph_builder_u10.py b/libs/openant-core/tests/parsers/ruby/test_call_graph_builder_u10.py new file mode 100644 index 00000000..50ab3ddc --- /dev/null +++ b/libs/openant-core/tests/parsers/ruby/test_call_graph_builder_u10.py @@ -0,0 +1,249 @@ +"""Regression tests for the Ruby call graph builder (u10 blind-fix batch). + +Five confirmed bugs in parsers/ruby/call_graph_builder.py, each driven through +the REAL pipeline (FunctionExtractor -> CallGraphBuilder) so the func_data shape +(indented `code`, `imports`, `class_name`, ...) matches production exactly. + + [1] parenless free-function call drops the edge (assert edge PRESENT) + [8] ClassName.method over-resolves to an unrelated file (assert edge ABSENT) + [11] user method named like a builtin gets filtered (assert edge PRESENT) + [31] m = method(:helper); m.call drops the edge (assert edge PRESENT) + [49] require 'auth' substring-matches authentication.rb (assert edge ABSENT) +""" + +import sys +import tempfile +from pathlib import Path + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +from parsers.ruby.function_extractor import FunctionExtractor +from parsers.ruby.call_graph_builder import CallGraphBuilder + + +def _build(files: dict) -> tuple[dict, CallGraphBuilder]: + """Write Ruby sources to a temp repo, run the real two-stage pipeline.""" + repo = Path(tempfile.mkdtemp()) + for name, src in files.items(): + p = repo / name + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(src) + extractor = FunctionExtractor(str(repo)) + output = extractor.extract_all(list(files.keys())) + builder = CallGraphBuilder(output) + builder.build_call_graph() + return output, builder + + +def _fid(output: dict, suffix: str) -> str: + matches = [k for k in output["functions"] if k.endswith(suffix)] + assert len(matches) == 1, f"expected one func id ending {suffix!r}, got {matches}" + return matches[0] + + +# ----------------------------------------------------------------- [1] parenless +def test_parenless_free_function_call_is_an_edge(): + """`main` body is a bare `greet` (no parens) — the main->greet edge must exist.""" + output, builder = _build( + {"app.rb": "def greet\n puts 'hi'\nend\n\ndef main\n greet\nend\n"} + ) + caller = _fid(output, ":main") + callee = _fid(output, ":greet") + assert callee in builder.call_graph[caller], ( + f"parenless call edge missing: {caller} -> {callee}; got {builder.call_graph[caller]}" + ) + + +def test_parenless_local_variable_is_not_an_edge(): + """Precision guard: a bare identifier that is a LOCAL VAR must NOT become a call. + + `greet` here is a method name AND a local variable in `main`; the bare + reference `greet` on the last line is a variable read, not a call. + """ + output, builder = _build( + { + "app.rb": "def greet\n puts 'hi'\nend\n\n" + "def main\n greet = 1\n greet\nend\n" + } + ) + caller = _fid(output, ":main") + callee = _fid(output, ":greet") + assert callee not in builder.call_graph[caller], ( + f"local variable wrongly treated as a call: {caller} -> {callee}" + ) + + +def test_parenless_unknown_identifier_is_not_an_edge(): + """Precision guard: a bare identifier that is NOT a known function is not a call.""" + output, builder = _build( + {"app.rb": "def greet\n puts 'hi'\nend\n\ndef main\n unknown_thing\nend\n"} + ) + caller = _fid(output, ":main") + # No edge to anything (unknown_thing is not a function). + assert builder.call_graph[caller] == [], ( + f"unknown bare identifier produced an edge: {builder.call_graph[caller]}" + ) + + +# ------------------------------------------------- [8] cross-file over-resolution +def test_class_call_does_not_resolve_to_unrelated_file(): + """Worker is defined in an unrelated file with no require link — no edge.""" + output, builder = _build( + { + "caller.rb": "class Caller\n def run\n Worker.process()\n end\nend\n", + "decoy.rb": "class Worker\n def process\n 42\n end\nend\n", + } + ) + caller = _fid(output, ":Caller.run") + forbidden = _fid(output, "decoy.rb:Worker.process") + assert forbidden not in builder.call_graph[caller], ( + f"over-resolved to unrelated file: {caller} -> {forbidden}; " + f"got {builder.call_graph[caller]}" + ) + + +def test_class_call_still_resolves_when_required(): + """Recall guard for [8]: when caller requires the defining file, edge stays.""" + output, builder = _build( + { + "caller.rb": "require_relative 'worker'\n" + "class Caller\n def run\n Worker.process()\n end\nend\n", + "worker.rb": "class Worker\n def process\n 42\n end\nend\n", + } + ) + caller = _fid(output, ":Caller.run") + callee = _fid(output, "worker.rb:Worker.process") + assert callee in builder.call_graph[caller], ( + f"required class-call edge lost: {caller} -> {callee}; got {builder.call_graph[caller]}" + ) + + +# --------------------------------------------------------------- [11] builtin leak +def test_user_method_named_like_builtin_is_an_edge(): + """`render` is in RUBY_BUILTINS but here it is a user function in the same file.""" + output, builder = _build( + {"app.rb": "def render\n 1\nend\n\ndef main\n render()\nend\n"} + ) + caller = _fid(output, ":main") + callee = _fid(output, ":render") + assert callee in builder.call_graph[caller], ( + f"user method named like builtin filtered: {caller} -> {callee}; " + f"got {builder.call_graph[caller]}" + ) + + +def test_genuine_builtin_not_linked_to_unrelated_user_method(): + """Scope guard for [11]: a genuine builtin call must NOT link to a same-named + user method in an UNRELATED file.""" + output, builder = _build( + { + "user.rb": "def main\n puts('hi')\nend\n", + "other.rb": "class Box\n def puts\n 99\n end\nend\n", + } + ) + caller = _fid(output, "user.rb:main") + forbidden = _fid(output, "other.rb:Box.puts") + assert forbidden not in builder.call_graph[caller], ( + f"genuine builtin linked to unrelated user method: {caller} -> {forbidden}" + ) + + +# ---------------------------------------------------------- [31] method-object var +def test_method_object_call_is_an_edge(): + """`m = method(:helper); m.call` — the caller_fn->helper edge must exist.""" + output, builder = _build( + { + "m.rb": "def helper\n 42\nend\n\n" + "def caller_fn\n m = method(:helper)\n m.call\nend\n" + } + ) + caller = _fid(output, ":caller_fn") + callee = _fid(output, ":helper") + assert callee in builder.call_graph[caller], ( + f"method-object edge missing: {caller} -> {callee}; got {builder.call_graph[caller]}" + ) + + +# ------------------------------------------- [31] single-unconditional GUARD +def test_method_object_reassignment_not_resolved(): + """GUARD (reassignment): `m = method(:a); m = method(:b); m.call` is + last-write-wins, so the binding is a "maybe". The guard must NOT assert a + definite edge to EITHER target — pinned behavior: no edge at all.""" + output, builder = _build( + { + "m.rb": "def a\n 1\nend\n\ndef b\n 2\nend\n\n" + "def caller_fn\n m = method(:a)\n m = method(:b)\n m.call\nend\n" + } + ) + caller = _fid(output, ":caller_fn") + a_id = _fid(output, "m.rb:a") + b_id = _fid(output, "m.rb:b") + edges = builder.call_graph[caller] + assert a_id not in edges and b_id not in edges, ( + f"reassigned method-object asserted a maybe-binding as definite: {edges}" + ) + + +def test_method_object_conditional_binding_not_resolved(): + """GUARD (conditional): a binding inside an `if`/`else` branch is not + unconditional, so `m.call` must NOT resolve (no edge to either target).""" + output, builder = _build( + { + "m.rb": "def a\n 1\nend\n\ndef b\n 2\nend\n\n" + "def caller_fn(cond)\n if cond\n m = method(:a)\n else\n" + " m = method(:b)\n end\n m.call\nend\n" + } + ) + caller = _fid(output, ":caller_fn") + a_id = _fid(output, "m.rb:a") + b_id = _fid(output, "m.rb:b") + edges = builder.call_graph[caller] + assert a_id not in edges and b_id not in edges, ( + f"conditional method-object resolved despite non-unconditional binding: {edges}" + ) + + +# ------------------------------------------------ [49] substring import over-match +def test_require_does_not_substring_match_longer_filename(): + """require 'auth' must NOT match authentication.rb by substring. + + A same-named `shared_helper` is also defined in a third file so the + global unique-name fallback (resolution step 4) CANNOT fire — that + isolates the substring-import mechanism (step 3) as the only thing + that could (wrongly) produce the forbidden edge. + """ + output, builder = _build( + { + "main.rb": "require 'auth'\ndef caller_fn\n shared_helper()\nend\n", + "authentication.rb": "def shared_helper\n 1\nend\n", + "elsewhere.rb": "def shared_helper\n 2\nend\n", + } + ) + caller = _fid(output, "main.rb:caller_fn") + forbidden = "authentication.rb:shared_helper" + assert forbidden not in builder.call_graph[caller], ( + f"require substring over-matched: {caller} -> {forbidden}; " + f"got {builder.call_graph[caller]}" + ) + + +def test_require_resolves_by_basename_equality(): + """Recall guard for [49]: require 'auth' DOES resolve to auth.rb (basename eq). + + A same-named decoy defeats the unique-name fallback, so a passing edge + must come from the basename-equality require match (step 3). + """ + output, builder = _build( + { + "main.rb": "require 'auth'\ndef caller_fn\n shared_helper()\nend\n", + "auth.rb": "def shared_helper\n 1\nend\n", + "elsewhere.rb": "def shared_helper\n 2\nend\n", + } + ) + caller = _fid(output, "main.rb:caller_fn") + callee = "auth.rb:shared_helper" + assert callee in builder.call_graph[caller], ( + f"basename-equal require edge lost: {caller} -> {callee}; " + f"got {builder.call_graph[caller]}" + ) diff --git a/libs/openant-core/tests/parsers/ruby/test_function_extractor_u11.py b/libs/openant-core/tests/parsers/ruby/test_function_extractor_u11.py new file mode 100644 index 00000000..2569933d --- /dev/null +++ b/libs/openant-core/tests/parsers/ruby/test_function_extractor_u11.py @@ -0,0 +1,140 @@ +"""Regression tests for the Ruby function_extractor (u11 blind-fix batch). + +Six confirmed bugs in parsers/ruby/function_extractor.py, each driven through the +REAL FunctionExtractor on a temp .rb file so the parse/extract shape matches +production exactly. Assertions are on the exported `functions` dict. + + [6] nested `def` inside a `def` body is never extracted (assert present) + [18] `def self.initialize` mis-typed 'constructor' (assert singleton_method + is_singleton) + [24] `define_method(:sym){}` not registered (assert sym present) + [42] `alias`/`alias_method` aliased name not registered (assert new name present) + [44] method after bare `private` not marked private (assert unit_type private_method) + [50] method in nested class gets bare class_name (assert class_name 'Outer::Inner') +""" + +import sys +import tempfile +from pathlib import Path + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +from parsers.ruby.function_extractor import FunctionExtractor + + +def _extract(src: str, name: str = "m.rb") -> dict: + """Write one Ruby source to a temp repo, run the real extractor, return functions.""" + repo = Path(tempfile.mkdtemp()) + (repo / name).write_text(src) + extractor = FunctionExtractor(str(repo)) + output = extractor.extract_all([name]) + return output["functions"] + + +def _find(functions: dict, suffix: str): + """Return the single func_data whose id ends with `suffix`, or None.""" + matches = [v for k, v in functions.items() if k.endswith(suffix)] + assert len(matches) <= 1, f"expected <=1 func id ending {suffix!r}, got {list(functions)}" + return matches[0] if matches else None + + +# ------------------------------------------------------------------ [6] nested def +def test_nested_def_is_extracted(): + """A `def inner` nested in `def outer`'s body must be extracted as a unit.""" + functions = _extract("def outer\n def inner\n 1\n end\nend\n") + assert _find(functions, ":inner") is not None, ( + f"nested def 'inner' missing; got {list(functions)}" + ) + # outer must still be present (no double-count regression) + assert _find(functions, ":outer") is not None + + +# ------------------------------------------------------ [18] self.initialize ordering +def test_self_initialize_is_singleton_not_constructor(): + """`def self.initialize` is a class singleton method, not the instance constructor.""" + functions = _extract( + "class Widget\n def self.initialize\n @count = 0\n end\nend\n", + "widget.rb", + ) + fn = _find(functions, ":Widget.initialize") + assert fn is not None, f"Widget.initialize missing; got {list(functions)}" + assert fn["is_singleton"] is True, f"expected is_singleton True; got {fn['is_singleton']}" + assert fn["unit_type"] == "singleton_method", ( + f"expected unit_type singleton_method; got {fn['unit_type']!r}" + ) + + +# ----------------------------------------------------------------- [24] define_method +def test_define_method_registers_symbol(): + """`define_method(:render){...}` must register `render` as a method unit.""" + functions = _extract( + 'def ctrl\n 1\nend\n\nclass Widget\n define_method(:render) do\n "x"\n end\nend\n' + ) + assert _find(functions, ":ctrl") is not None, "in-file control 'ctrl' missing (file parse)" + fn = _find(functions, ":render") or _find(functions, ".render") + assert fn is not None, f"define_method 'render' missing; got {list(functions)}" + assert fn["name"] == "render", f"expected name 'render'; got {fn['name']!r}" + assert fn["class_name"] == "Widget", f"expected class_name Widget; got {fn['class_name']!r}" + + +# ------------------------------------------------------------------------ [42] alias +def test_alias_keyword_registers_new_name(): + """`alias greet hello` must register `greet` as a method node.""" + functions = _extract( + 'def control\n 1\nend\nclass Greeter\n def hello\n "hi"\n end\n' + " alias greet hello\nend\n" + ) + assert _find(functions, ":Greeter.hello") is not None, "control method 'hello' missing" + fn = _find(functions, ":Greeter.greet") + assert fn is not None, f"aliased name 'greet' missing; got {list(functions)}" + assert fn["name"] == "greet" + assert fn["class_name"] == "Greeter" + + +def test_alias_method_call_registers_new_name(): + """`alias_method :greet, :hello` must register `greet` (distinct AST node from `alias`).""" + functions = _extract( + 'class Greeter\n def hello\n "hi"\n end\n alias_method :greet, :hello\nend\n' + ) + fn = _find(functions, ":Greeter.greet") + assert fn is not None, f"alias_method name 'greet' missing; got {list(functions)}" + assert fn["name"] == "greet" + + +# ------------------------------------------------------------------- [44] private kw +def test_method_after_private_keyword_is_private(): + """A method following a bare `private` keyword must be unit_type private_method.""" + functions = _extract( + "class Foo\n private\n\n def secret\n 42\n end\nend\n", "foo.rb" + ) + fn = _find(functions, ":Foo.secret") + assert fn is not None, f"Foo.secret missing; got {list(functions)}" + assert fn["unit_type"] == "private_method", ( + f"expected unit_type private_method; got {fn['unit_type']!r}" + ) + + +def test_method_before_private_keyword_stays_public(): + """Precision guard: a method declared BEFORE `private` must stay public ('method').""" + functions = _extract( + "class Foo\n def open_api\n 1\n end\n\n private\n\n def secret\n 2\n end\nend\n", + "foo2.rb", + ) + pub = _find(functions, ":Foo.open_api") + assert pub is not None and pub["unit_type"] == "method", ( + f"public method mis-typed: {pub['unit_type'] if pub else None}" + ) + + +# ----------------------------------------------------------- [50] nested class_name +def test_nested_class_method_has_composed_class_name(): + """A method in `class Inner` nested in `class Outer` must have class_name 'Outer::Inner'.""" + functions = _extract( + "class Outer\n class Inner\n def deep_method\n 1\n end\n end\nend\n", + "n.rb", + ) + fn = _find(functions, ".deep_method") + assert fn is not None, f"deep_method missing; got {list(functions)}" + assert fn["class_name"] == "Outer::Inner", ( + f"expected class_name 'Outer::Inner'; got {fn['class_name']!r}" + ) diff --git a/libs/openant-core/tests/parsers/ruby/test_ruby_conditional_collision.py b/libs/openant-core/tests/parsers/ruby/test_ruby_conditional_collision.py new file mode 100644 index 00000000..bc679283 --- /dev/null +++ b/libs/openant-core/tests/parsers/ruby/test_ruby_conditional_collision.py @@ -0,0 +1,49 @@ +"""Bug 1 (Ruby): same-name defs in mutually-exclusive conditional branches must +NOT collapse to the last-in-source one. + +Verified harmful (independent + judge, real `ruby` interpreter): for + if COND then def k; A end else def k; B end end +only the taken branch's `def` executes — and it may be the EARLIER (if) branch, +but the extractor kept only the later (else) one. Both branches are +env-dependently reachable, so the fix keeps BOTH via the `#L` +disambiguation the Python extractor already uses. Collision-only: a unique name +keeps its clean id. (Unconditional reopening stays correct — last-wins — and is +simply also kept-both, the same tradeoff Python accepts.) +""" +import sys +from pathlib import Path + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +from parsers.ruby.function_extractor import FunctionExtractor + + +def _extract(tmp_path: Path, filename: str, source: str) -> dict: + (tmp_path / filename).write_text(source) + return FunctionExtractor(str(tmp_path)).extract_all([filename]) + + +def _named(out: dict, name: str): + return [fid for fid, d in out["functions"].items() if d.get("name") == name] + + +def test_conditional_ifelse_keeps_both_branches(tmp_path): + src = ( + "if ENV['X']\n" + " def k; if_branch; end\n" + "else\n" + " def k; else_branch; end\n" + "end\n" + ) + out = _extract(tmp_path, "cond.rb", src) + ks = _named(out, "k") + assert len(ks) == 2, f"both conditional branches of k must be kept; got {ks}" + bodies = " ".join(out["functions"][f]["code"] for f in ks) + assert "if_branch" in bodies and "else_branch" in bodies, f"a branch was dropped: {bodies}" + + +def test_unique_name_id_unchanged(tmp_path): + out = _extract(tmp_path, "u.rb", "def solo; 1; end\n") + ids = _named(out, "solo") + assert ids == ["u.rb:solo"], f"unique-name id must stay byte-identical; got {ids}" diff --git a/libs/openant-core/tests/parsers/test_entry_point_detector_u12.py b/libs/openant-core/tests/parsers/test_entry_point_detector_u12.py new file mode 100644 index 00000000..1c6f4ec3 --- /dev/null +++ b/libs/openant-core/tests/parsers/test_entry_point_detector_u12.py @@ -0,0 +1,110 @@ +"""Tests for EntryPointDetector — a silent binary `main` (C/Go) classified +unit_type='main' must be seeded as a reachability entry point. + +Regression for the gap where `main` was missing from ENTRY_POINT_TYPES: the C +and Go extractors correctly classify a program's `main` function as +unit_type='main', but the detector only honored unit types in +ENTRY_POINT_TYPES. A *silent* main (no user-input pattern, no decorator, not +module_level) therefore produced zero entry-point reasons, was never seeded as +an execution root, and every function it transitively calls was falsely marked +unreachable (reachability blackout on CLI/binary programs). + +A program's `main` is an execution root by definition; over-approximating it as +an entry point is safe (a false-unreachable hides exploitable code), and a +library has no `main`, so this does not over-claim. +""" +import sys +from pathlib import Path + +# tests/parsers/ -> parents[2] == libs/openant-core (the dir containing utilities/) +_CORE_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(_CORE_ROOT)) + +from utilities.agentic_enhancer.entry_point_detector import ( # noqa: E402 + ENTRY_POINT_TYPES, + EntryPointDetector, +) + + +def _make_detector(func_id: str, func_data: dict) -> EntryPointDetector: + return EntryPointDetector({func_id: func_data}, call_graph={}) + + +def test_main_is_in_entry_point_types(): + """`main` must be a recognized entry-point unit type so the reachability + filter treats a program's execution root as a seed.""" + assert "main" in ENTRY_POINT_TYPES, ( + "'main' must be in ENTRY_POINT_TYPES so a function classified " + "unit_type='main' is seeded as a reachability entry point" + ) + + +def test_silent_c_main_is_entry_point(): + """A silent C `main` (no user-input pattern, no decorator) classified + unit_type='main' must be detected as an entry point.""" + detector = _make_detector( + "main.c:main", + { + "name": "main", + "unit_type": "main", + "code": "int main(void) { helper(); return 0; }", + "decorators": [], + }, + ) + entry_points = detector.detect_entry_points() + assert "main.c:main" in entry_points, ( + "silent C main was filtered out — its callees become falsely unreachable" + ) + + +def test_silent_go_main_is_entry_point(): + """A silent Go `main` classified unit_type='main' must be detected as an + entry point (language-agnostic: same unit_type, same seeding).""" + detector = _make_detector( + "main.go:main", + { + "name": "main", + "unit_type": "main", + "code": "func main() { helper() }", + "decorators": [], + }, + ) + entry_points = detector.detect_entry_points() + assert "main.go:main" in entry_points, ( + "silent Go main was filtered out — its callees become falsely unreachable" + ) + + +def test_main_by_name_is_entry_point(): + """Defensive: a function named `main` is seeded as an entry point even if + the extractor classified its unit_type as something other than 'main' + (e.g. a generic 'function').""" + detector = _make_detector( + "main.c:main", + { + "name": "main", + "unit_type": "function", + "code": "int main(void) { helper(); return 0; }", + "decorators": [], + }, + ) + entry_points = detector.detect_entry_points() + assert "main.c:main" in entry_points, ( + "a function named main must be seeded as an execution root by name" + ) + + +def test_non_main_silent_function_is_not_entry_point(): + """True-negative anchor: an ordinary silent helper (no main name, no entry + unit_type, no input pattern) must NOT be an entry point.""" + detector = _make_detector( + "main.c:helper", + { + "name": "helper", + "unit_type": "function", + "code": "void helper(void) { return; }", + "decorators": [], + }, + ) + entry_points = detector.detect_entry_points() + assert "main.c:helper" not in entry_points diff --git a/libs/openant-core/tests/parsers/test_repository_scanner_is_test_file_cma.py b/libs/openant-core/tests/parsers/test_repository_scanner_is_test_file_cma.py new file mode 100644 index 00000000..ee65a613 --- /dev/null +++ b/libs/openant-core/tests/parsers/test_repository_scanner_is_test_file_cma.py @@ -0,0 +1,117 @@ +"""Cross-language regression: repository_scanner test-file classification must be anchored. + +Bug (path_substring_exclusion family, bundle entries [2] multi-lang + [22] zig): + `is_test_file` (c/php/python/ruby) and `_is_test_file`/`_is_test_directory` (zig) + classified a file/dir as a TEST using an UNANCHORED substring match + (`for pattern in test_patterns: if pattern in path_lower`). Because `test_` is a + substring of `latest_`/`greatest_`/`contest_`, and zig's bare `test`/`spec` tokens + are substrings of `latest`/`contest`/`attestation`/`inspector`, real source files + whose name merely CONTAINS a test token were silently classified as tests and + DROPPED from extraction (default `skip_tests=True`). + +Fix shape (one mechanism across all 5 langs): anchor the match to whole PATH +COMPONENTS (a directory part == test/tests/spec/specs) OR basename conventions +(`test_*`, `*_test.`, `*_spec.`, `*Test.`, `conftest.py`, etc.). + +This test drives each scanner's classification predicate directly: + - DECOY case: a real source whose name CONTAINS a token as substring -> NOT a test. + - POSITIVE case: a genuine test file/dir -> still IS a test (don't over-narrow). + +JS (`isTestFile` regex) and Go (`HasSuffix "_test.go"`) are already anchored and are +intentionally NOT exercised here. +""" + +import sys +from pathlib import Path + +import pytest + +_CORE_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(_CORE_ROOT)) + + +def _make_scanner(lang, repo_path="/tmp/repo"): + """Instantiate each language's RepositoryScanner with skip_tests on.""" + if lang == "c": + from parsers.c.repository_scanner import RepositoryScanner + return RepositoryScanner(repo_path, {"skip_tests": True}) + if lang == "php": + from parsers.php.repository_scanner import RepositoryScanner + return RepositoryScanner(repo_path, {"skip_tests": True}) + if lang == "python": + from parsers.python.repository_scanner import RepositoryScanner + return RepositoryScanner(repo_path, {"skip_tests": True}) + if lang == "ruby": + from parsers.ruby.repository_scanner import RepositoryScanner + return RepositoryScanner(repo_path, {"skip_tests": True}) + if lang == "zig": + from parsers.zig.repository_scanner import RepositoryScanner + return RepositoryScanner(repo_path, skip_tests=True) + raise ValueError(lang) + + +def _is_test(lang, scanner, relative_path): + """Call the per-language test-file classification predicate.""" + if lang == "zig": + return scanner._is_test_file(relative_path) + return scanner.is_test_file(relative_path) + + +# (lang, decoy_real_source_that_must_NOT_be_a_test) +DECOYS = [ + ("c", "latest_dir/main.c"), + ("c", "contest/sol.c"), + ("c", "src/protest.c"), + ("php", "src/protest_api.php"), + ("php", "app/latest_controller.php"), + ("python", "pkg/latest.py"), + ("python", "pkg/greatest_helper.py"), + ("ruby", "lib/latest_x.rb"), + ("ruby", "lib/contest.rb"), + ("zig", "src/latest.zig"), + ("zig", "src/contest.zig"), + ("zig", "src/attestation.zig"), + ("zig", "inspector/foo.zig"), +] + +# (lang, genuine_test_file_that_MUST_still_be_classified_as_a_test) +POSITIVES = [ + ("c", "tests/test_foo.c"), + ("c", "src/foo_test.c"), + ("php", "tests/FooTest.php"), + ("php", "src/test_helper.php"), + ("python", "tests/test_foo.py"), + ("python", "pkg/conftest.py"), + ("ruby", "spec/foo_spec.rb"), + ("ruby", "test/test_foo.rb"), + ("zig", "test/foo.zig"), + ("zig", "src/foo_test.zig"), +] + + +@pytest.mark.parametrize("lang,relative_path", DECOYS) +def test_decoy_real_source_not_classified_as_test(lang, relative_path): + scanner = _make_scanner(lang) + assert not _is_test(lang, scanner, relative_path), ( + f"{lang}: real source {relative_path!r} wrongly classified as a test " + f"(unanchored substring match)" + ) + + +@pytest.mark.parametrize("lang,relative_path", POSITIVES) +def test_positive_genuine_test_still_classified(lang, relative_path): + scanner = _make_scanner(lang) + assert _is_test(lang, scanner, relative_path), ( + f"{lang}: genuine test {relative_path!r} must still be classified as a test" + ) + + +def test_zig_test_directory_decoy_not_excluded(): + """zig dir-level: `inspector`/`latest` dirs must NOT be treated as test dirs.""" + scanner = _make_scanner("zig") + assert not scanner._is_test_directory("inspector") + assert not scanner._is_test_directory("latest_dir") + # positive: a real `test` dir IS a test dir + assert scanner._is_test_directory("test") + assert scanner._is_test_directory("tests") + assert scanner._is_test_directory("spec") diff --git a/libs/openant-core/tests/parsers/zig/test_call_graph_builder_u13.py b/libs/openant-core/tests/parsers/zig/test_call_graph_builder_u13.py new file mode 100644 index 00000000..e5955c54 --- /dev/null +++ b/libs/openant-core/tests/parsers/zig/test_call_graph_builder_u13.py @@ -0,0 +1,132 @@ +"""Regression tests for the Zig call graph builder (u13). + +Three confirmed call-graph recall bugs, each reproduced through the REAL +extractor -> builder pipeline (FunctionExtractor.extract() feeding +CallGraphBuilder(...).build()), asserting the dropped edge is present. + +- [BUG 3] local-type dispatch: `const o = Foo{}; o.method()` (and the + direct `Foo{}.method()`) produces no `caller -> method` edge, + because call-name extraction never recognises a tree-sitter + `field_expression` callee, so the method name is never emitted. +- [BUG 17] builtin-filter leak: a user-defined fn whose name collides with + a ZIG_BUILTINS entry (e.g. `expect`) is dropped by the builtin + filter before resolution, even though a same-file user function + of that name exists. +- [BUG 41] const-alias dataflow: `const f = handler; f()` loses the edge to + `handler`, because the name index maps only fn-decl names, never + the simple const alias binding. +""" + +import os +import sys +import tempfile +from pathlib import Path + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +from parsers.zig.function_extractor import FunctionExtractor +from parsers.zig.call_graph_builder import CallGraphBuilder + + +def _run_pipeline(src: str) -> dict: + """Run the real extractor -> builder pipeline on a single zig source file.""" + workdir = tempfile.mkdtemp() + file_path = os.path.join(workdir, "m.zig") + with open(file_path, "w") as fh: + fh.write(src) + scan_results = {"files": [{"path": "m.zig"}]} + extractor_output = FunctionExtractor(workdir, scan_results).extract() + return CallGraphBuilder(extractor_output).build() + + +def test_bug3_local_type_dispatch_method_call_edge(): + """`const o = C{}; o.m()` must yield an `f -> C.m` call-graph edge. + + Note: the target id is the QUALIFIED `m.zig:C.m`. Prior to the u14 [BUG 37] + fix, struct methods were (incorrectly) emitted under their bare name, so this + assertion read `m.zig:m`. The method is now correctly keyed by its qualified + `Container.method` id; the edge itself is unchanged. + """ + src = ( + "const C = struct { fn m(self: C) i32 { _ = self; return 1; } };\n" + "fn f() i32 { const o = C{}; return o.m(); }\n" + ) + cg = _run_pipeline(src)["call_graph"] + assert "m.zig:C.m" in cg.get("m.zig:f", []), ( + f"Expected f -> C.m method-call edge, got call_graph={cg}" + ) + + +def test_bug3_direct_struct_init_method_call_edge(): + """The direct `C{}.m()` form must also yield an `f -> C.m` edge. + + See the qualified-id note on test_bug3_local_type_dispatch_method_call_edge. + """ + src = ( + "const C = struct { fn m(self: C) i32 { _ = self; return 1; } };\n" + "fn f() i32 { return C{}.m(); }\n" + ) + cg = _run_pipeline(src)["call_graph"] + assert "m.zig:C.m" in cg.get("m.zig:f", []), ( + f"Expected f -> C.m direct-init method-call edge, got call_graph={cg}" + ) + + +def test_bug17_user_fn_shadowing_builtin_is_not_filtered(): + """A user fn named `expect` (a ZIG_BUILTINS name) must keep its edge.""" + src = ( + "fn expect(ok: bool) void {\n" + " _ = ok;\n" + "}\n" + "\n" + "fn main() void {\n" + " expect(true);\n" + "}\n" + ) + cg = _run_pipeline(src)["call_graph"] + assert "m.zig:expect" in cg.get("m.zig:main", []), ( + f"Expected main -> expect edge (user fn shadows builtin), got call_graph={cg}" + ) + + +def test_bug17_genuine_builtin_call_is_still_filtered(): + """Scope guard: a builtin call with NO same-file user fn stays filtered. + + `@import` is a genuine builtin and there is no user `@import` function, + so it must not appear as an edge — the fix only un-filters builtins that + are shadowed by a same-file user definition. + """ + src = ( + "fn main() void {\n" + " const std = @import(\"std\");\n" + " _ = std;\n" + "}\n" + ) + cg = _run_pipeline(src)["call_graph"] + # No user fn named @import / import exists, so main has no resolvable edge. + assert cg.get("m.zig:main", []) == [], ( + f"Genuine builtin call should not produce an edge, got call_graph={cg}" + ) + + +def test_bug41_const_alias_call_edge(): + """`const f = handler; f()` must yield a `viaAlias -> handler` edge.""" + src = ( + "fn handler() void {}\n" + "fn viaAlias() void {\n" + " const f = handler;\n" + " f();\n" + "}\n" + "fn direct() void {\n" + " handler();\n" + "}\n" + ) + cg = _run_pipeline(src)["call_graph"] + assert "m.zig:handler" in cg.get("m.zig:viaAlias", []), ( + f"Expected viaAlias -> handler alias edge, got call_graph={cg}" + ) + # Control: the direct call must keep working too. + assert "m.zig:handler" in cg.get("m.zig:direct", []), ( + f"Direct call edge regressed, got call_graph={cg}" + ) diff --git a/libs/openant-core/tests/parsers/zig/test_function_extractor_generic_container.py b/libs/openant-core/tests/parsers/zig/test_function_extractor_generic_container.py new file mode 100644 index 00000000..372c7751 --- /dev/null +++ b/libs/openant-core/tests/parsers/zig/test_function_extractor_generic_container.py @@ -0,0 +1,95 @@ +"""Regression test for the Zig generic-container method-attribution bug. + +Zig's idiomatic generic container is a type-returning function: + pub fn List(comptime T: type) type { return struct { pub fn push(...) ... }; } +The returned struct is anonymous in the AST (a `struct_declaration` reached via +`return_expression`), NOT a `const Name = struct {...}` (`variable_declaration`). +The walker only threaded struct context for the variable_declaration form, so methods +inside a type-returning container were emitted as bare top-level functions with +class_name=None — and two distinct containers' same-named methods collided on one id. + +Driven through the REAL extractor (FunctionExtractor.extract()) on a temp .zig file. +""" + +import os +import sys +import tempfile +from pathlib import Path + +import pytest + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +from parsers.zig.function_extractor import FunctionExtractor + + +def _extract(src: str) -> dict: + workdir = tempfile.mkdtemp() + with open(os.path.join(workdir, "m.zig"), "w") as fh: + fh.write(src) + return FunctionExtractor(workdir, {"files": [{"path": "m.zig"}]}).extract() + + +def _zig_parser_is_grammar_aligned() -> bool: + """Probe the PREREQUISITE behavior (not this fix's): does a *named* struct's method + extract as Container.method? That capability is provided by the tree-sitter-zig + grammar-alignment work (>=1.1.2 node names struct_declaration/variable_declaration; + PRs 87/110, commit 322920e), independent of the generic-container fix under test. + On a base whose parser still matches stale node names (VarDecl/container_decl), no + struct methods extract at all, so these tests cannot pass for reasons unrelated to + the fix.""" + probe = "const _Probe = struct {\n pub fn _m(self: _Probe) void { _ = self; }\n};\n" + return "m.zig:_Probe._m" in _extract(probe)["functions"] + + +# Skip (not fail) with an explanatory message when run on a base that lacks the +# grammar-alignment prerequisite — so a human or agent running this on raw master sees +# *why* instead of a cryptic assertion failure. Supported base: staging/parser-fix-stack, +# which carries upstream PR #110 (Zig parser realignment) AND the tree-sitter-zig>=1.1.2 +# grammar pin. This is NOT landable on master standalone. +pytestmark = pytest.mark.skipif( + not _zig_parser_is_grammar_aligned(), + reason=( + "Zig parser not grammar-aligned (needs tree-sitter-zig>=1.1.2 node names " + "struct_declaration/variable_declaration, from upstream PR #110 + the grammar " + "pin). On such a base no struct methods extract, so the generic-container fix " + "cannot pass. Supported base: staging/parser-fix-stack — not landable on master." + ), +) + + +def test_generic_container_method_qualified_to_container(): + src = ( + "pub fn List(comptime T: type) type {\n" + " return struct {\n" + " pub fn push(self: *@This(), x: T) void { _ = self; _ = x; }\n" + " };\n" + "}\n" + "fn ordinary() void {}\n" + ) + out = _extract(src) + funcs = out["functions"] + assert "m.zig:List.push" in funcs, f"List.push missing; keys = {sorted(funcs)}" + info = funcs["m.zig:List.push"] + assert info["class_name"] == "List" + assert info["qualified_name"] == "List.push" + assert info["unit_type"] == "method" + # The method must NOT leak as a bare top-level function. + assert "m.zig:push" not in funcs, f"unqualified push leaked: {sorted(funcs)}" + # The plain function is unaffected. + assert "m.zig:ordinary" in funcs, sorted(funcs) + + +def test_two_generic_containers_methods_no_collision(): + src = ( + "pub fn List(comptime T: type) type {\n" + " return struct { pub fn len(self: *@This()) usize { _ = self; return 0; } };\n" + "}\n" + "pub fn Ring(comptime T: type) type {\n" + " return struct { pub fn len(self: *@This()) usize { _ = self; return 1; } };\n" + "}\n" + ) + funcs = _extract(src)["functions"] + assert "m.zig:List.len" in funcs, f"keys = {sorted(funcs)}" + assert "m.zig:Ring.len" in funcs, f"silent collision/data-loss; keys = {sorted(funcs)}" diff --git a/libs/openant-core/tests/parsers/zig/test_function_extractor_u14.py b/libs/openant-core/tests/parsers/zig/test_function_extractor_u14.py new file mode 100644 index 00000000..68bf4981 --- /dev/null +++ b/libs/openant-core/tests/parsers/zig/test_function_extractor_u14.py @@ -0,0 +1,145 @@ +"""Regression tests for the Zig FunctionExtractor (u14). + +Three confirmed extraction/metadata bugs, each reproduced through the REAL +extractor (FunctionExtractor.extract()) on a temp .zig file, asserting on the +emitted `functions` map. + +- [BUG 16] fn-name extraction: a free fn whose return type is a bare named + identifier (`fn makeWidget(v: i32) Widget { ... }`) is recorded + under the RETURN-TYPE name (`Widget`) because the identifier loop + overwrites `name` with the second `identifier` child (the return + type). The real fn name must be the FIRST identifier. +- [BUG 26] test classification: `pub fn testConnection() bool {}` is wrongly + classified `test` because `_classify_function` matches the + `startswith("test")` prefix. A plain function named testXxx is a + regular function, not a zig `test "..." {}` block. +- [BUG 37] struct/enum container methods: methods inside a + `const Foo = struct { fn method() ... };` container are never + extracted, because the walker keys on node types tree-sitter-zig + never emits (`VarDecl`/`container_decl`). The real types are + `variable_declaration` and `struct_declaration` / `enum_declaration` + / `union_declaration` / `opaque_declaration`. +""" + +import os +import sys +import tempfile +from pathlib import Path + +_CORE_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(_CORE_ROOT)) + +from parsers.zig.function_extractor import FunctionExtractor + + +def _extract(src: str) -> dict: + """Run the real extractor on a single zig source file; return extract() output.""" + workdir = tempfile.mkdtemp() + file_path = os.path.join(workdir, "m.zig") + with open(file_path, "w") as fh: + fh.write(src) + scan_results = {"files": [{"path": "m.zig"}]} + return FunctionExtractor(workdir, scan_results).extract() + + +# --------------------------------------------------------------------------- +# [BUG 16] fn name must be the first identifier, not the return-type identifier +# --------------------------------------------------------------------------- + +def test_bug16_named_return_type_does_not_shadow_fn_name(): + """`fn makeWidget(v: i32) Widget {}` must be recorded as makeWidget, not Widget.""" + src = ( + "const Widget = struct { x: i32 };\n\n" + "pub fn makeWidget(v: i32) Widget {\n" + " return Widget{ .x = v };\n" + "}\n" + ) + out = _extract(src) + funcs = out["functions"] + assert "m.zig:makeWidget" in funcs, ( + f"makeWidget missing; functions keys = {sorted(funcs)}" + ) + assert funcs["m.zig:makeWidget"]["name"] == "makeWidget" + # The return-type identifier must NOT have become a phantom function. + assert "m.zig:Widget" not in funcs, ( + f"return-type Widget leaked as a function; keys = {sorted(funcs)}" + ) + + +def test_bug16_generic_named_return_type_build_T(): + """Re-confirm the general case: `fn build() T` records build, not T.""" + src = "fn build() T {\n return undefined;\n}\n" + out = _extract(src) + funcs = out["functions"] + assert "m.zig:build" in funcs, f"build missing; keys = {sorted(funcs)}" + assert funcs["m.zig:build"]["name"] == "build" + assert "m.zig:T" not in funcs + + +# --------------------------------------------------------------------------- +# [BUG 26] a plain fn named testXxx must classify as 'function', not 'test' +# --------------------------------------------------------------------------- + +def test_bug26_fn_named_testconnection_is_function_not_test(): + """`pub fn testConnection() bool {}` must have unit_type 'function'.""" + src = "pub fn testConnection() bool {\n return true;\n}\n" + out = _extract(src) + funcs = out["functions"] + assert "m.zig:testConnection" in funcs, f"keys = {sorted(funcs)}" + assert funcs["m.zig:testConnection"]["unit_type"] == "function", ( + f"testConnection wrongly classified: " + f"{funcs['m.zig:testConnection']['unit_type']!r}" + ) + + +# --------------------------------------------------------------------------- +# [BUG 37] container methods (struct/enum/union/opaque) must be extracted +# under the qualified Container.method name. +# --------------------------------------------------------------------------- + +def test_bug37_struct_method_qualified_name_extracted(): + """`const Foo = struct { fn method() ... };` must yield Foo.method.""" + src = ( + "pub fn ordinary() void {}\n\n" + "const Foo = struct {\n" + " pub fn method(self: Foo) void {}\n" + "};\n" + ) + out = _extract(src) + funcs = out["functions"] + assert "m.zig:Foo.method" in funcs, ( + f"Foo.method missing; keys = {sorted(funcs)}" + ) + info = funcs["m.zig:Foo.method"] + assert info["qualified_name"] == "Foo.method" + assert info["class_name"] == "Foo" + assert info["unit_type"] == "method" + # The struct itself should be recorded as a class. + assert "m.zig:Foo" in out["classes"], f"classes = {sorted(out['classes'])}" + + +def test_bug37_enum_method_extracted(): + """`const E = enum { a, fn em() ... };` must yield E.em.""" + src = "const E = enum {\n a,\n pub fn em() void {}\n};\n" + out = _extract(src) + funcs = out["functions"] + assert "m.zig:E.em" in funcs, f"keys = {sorted(funcs)}" + assert funcs["m.zig:E.em"]["class_name"] == "E" + + +def test_bug37_union_method_extracted(): + """`const U = union(enum) { a: u8, fn um() ... };` must yield U.um.""" + src = "const U = union(enum) {\n a: u8,\n pub fn um() void {}\n};\n" + out = _extract(src) + funcs = out["functions"] + assert "m.zig:U.um" in funcs, f"keys = {sorted(funcs)}" + assert funcs["m.zig:U.um"]["class_name"] == "U" + + +def test_bug37_opaque_method_extracted(): + """`const O = opaque { fn om() ... };` must yield O.om.""" + src = "const O = opaque {\n pub fn om() void {}\n};\n" + out = _extract(src) + funcs = out["functions"] + assert "m.zig:O.om" in funcs, f"keys = {sorted(funcs)}" + assert funcs["m.zig:O.om"]["class_name"] == "O" diff --git a/libs/openant-core/tests/parsers/zig/test_zig_main_classification.py b/libs/openant-core/tests/parsers/zig/test_zig_main_classification.py index 0c3cdbde..49539ae3 100644 --- a/libs/openant-core/tests/parsers/zig/test_zig_main_classification.py +++ b/libs/openant-core/tests/parsers/zig/test_zig_main_classification.py @@ -56,4 +56,7 @@ def test_zig_other_classifications_unchanged(): # Guard against over-broadening the fix. assert _classify("init") == "constructor" assert _classify("create") == "constructor" - assert _classify("testThing") == "test" + # u14 (PR #110) anchored test detection to the underscore convention: + # camelCase `testThing` is an ordinary function, `test_thing` is a test. + assert _classify("test_thing") == "test" + assert _classify("testThing") == "function" diff --git a/libs/openant-core/tests/test_blackout_warning.py b/libs/openant-core/tests/test_blackout_warning.py new file mode 100644 index 00000000..9f2d7b3f --- /dev/null +++ b/libs/openant-core/tests/test_blackout_warning.py @@ -0,0 +1,61 @@ +"""Fix B — reachability blackout warning (advisory; never changes filtering). + +The #75 zero-seed net only fires at EXACTLY 0 entry points. A library like +tree-sitter trips a handful of INCIDENTAL seeds (code that merely contains an +input-reading pattern), yielding a 96.6% reduction that looks like a successful +filter while the real public-API core was dropped. `blackout_warning` catches +both the total blackout and this partial-blackout-with-only-incidental-seeds case, +and stays silent for a normal app (real route/main/CLI seeds, moderate reduction). +""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # libs/openant-core + +from utilities.agentic_enhancer import blackout_warning # noqa: E402 + + +def _details(*reason_lists): + """Build an entry_point_details-shaped dict from per-seed reason lists.""" + return {f"f{i}": {"reasons": rs} for i, rs in enumerate(reason_lists)} + + +def test_total_blackout_warns(): + assert blackout_warning(_details(), original_count=500, reachable_count=0) is not None + + +def test_partial_blackout_incidental_seeds_warns(): + # tree-sitter shape: 4 incidental input_pattern seeds, 712 -> 24 (96.6% pruned). + details = _details(["input_pattern:fopen"], ["input_pattern:read"], + ["input_pattern:getenv"], ["input_pattern:scanf"]) + assert blackout_warning(details, original_count=712, reachable_count=24) is not None + + +def test_structural_seed_suppresses_even_high_reduction(): + # A real CLI/main seed means the high reduction is legitimate, not a blackout. + details = _details(["unit_type:main"], ["input_pattern:read"]) + assert blackout_warning(details, original_count=712, reachable_count=24) is None + + +def test_normal_app_reduction_is_silent(): + # Arkime C shape: route/main seeds, 1655 -> 608 (63% pruned). No warning. + details = _details(["unit_type:cli_handler"], ["unit_type:main"], ["unit_type:http_handler"]) + assert blackout_warning(details, original_count=1655, reachable_count=608) is None + + +def test_decorator_and_name_seeds_are_structural(): + assert blackout_warning(_details(["decorator:@app.route"]), + original_count=712, reachable_count=24) is None + assert blackout_warning(_details(["name:main"]), + original_count=712, reachable_count=24) is None + + +def test_library_mode_suppresses_warning(): + # With library-mode on, a high reduction is the intended precise result. + details = _details(["input_pattern:read"]) + assert blackout_warning(details, original_count=712, reachable_count=24, + library_mode=True) is None + + +def test_empty_dataset_no_warning(): + assert blackout_warning(_details(), original_count=0, reachable_count=0) is None diff --git a/libs/openant-core/tests/test_c_macro_alias_cycle.py b/libs/openant-core/tests/test_c_macro_alias_cycle.py new file mode 100644 index 00000000..7e53b5aa --- /dev/null +++ b/libs/openant-core/tests/test_c_macro_alias_cycle.py @@ -0,0 +1,51 @@ +"""Regression test for F4: C call-graph builder must not crash on cyclic macro aliases. + +A function-like ``#define`` pair such as:: + + #define A(x) B(x) + #define B(x) A(x) + +produces ``macro_aliases = {"A": "B", "B": "A"}``. Before the fix, +``_resolve_call`` recursed on the aliased name with no cycle guard, so +resolving ``A`` recursed A->B->A->... until ``RecursionError`` aborted the +entire repository's C call-graph build. +""" + +from __future__ import annotations + +import pytest + +tree_sitter_c = pytest.importorskip("tree_sitter_c") + +from parsers.c.call_graph_builder import CallGraphBuilder + + +def _builder(macro_aliases): + return CallGraphBuilder( + { + "functions": {}, + "includes": {}, + "macros": {}, + "macro_aliases": macro_aliases, + } + ) + + +def test_cyclic_macro_alias_does_not_recurse(): + """Two-node alias cycle must resolve to None, not raise RecursionError.""" + builder = _builder({"A": "B", "B": "A"}) + # Must not raise; unresolved cyclic alias returns None. + assert builder._resolve_call("A", "foo.c") is None + assert builder._resolve_call("B", "foo.c") is None + + +def test_longer_macro_alias_cycle_terminates(): + """A 3-node alias cycle must also terminate.""" + builder = _builder({"A": "B", "B": "C", "C": "A"}) + assert builder._resolve_call("A", "foo.c") is None + + +def test_self_macro_alias_does_not_recurse(): + """A self-alias is short-circuited by the != guard but must stay safe.""" + builder = _builder({"A": "A"}) + assert builder._resolve_call("A", "foo.c") is None diff --git a/libs/openant-core/tests/test_entry_point_input_pattern_boundary.py b/libs/openant-core/tests/test_entry_point_input_pattern_boundary.py new file mode 100644 index 00000000..5de94f8e --- /dev/null +++ b/libs/openant-core/tests/test_entry_point_input_pattern_boundary.py @@ -0,0 +1,52 @@ +"""Regression test for F9: USER_INPUT_PATTERNS FastAPI alternation needs a word boundary. + +The pattern ``(Query|Body|Form|File|Header|Cookie)\\s*\\(`` (no leading ``\\b``) +matches any identifier *ending* in one of those words, so ordinary library +calls like ``setCookie(``, ``PQsendQuery(`` or ``getHeader(`` were flagged as +user-input sources and seeded as false remote-web entry points across C/Go/ +PHP/Python repos. The fix anchors the alternation with ``\\b`` so only the +standalone FastAPI dependency symbols match. +""" + +from __future__ import annotations + +import re + +from utilities.agentic_enhancer.entry_point_detector import USER_INPUT_PATTERNS + + +def _fastapi_pattern(): + """The FastAPI Query/Body/.../Cookie alternation from USER_INPUT_PATTERNS.""" + for p in USER_INPUT_PATTERNS: + if "Cookie" in p and "Query" in p: + return re.compile(p) + raise AssertionError("FastAPI input pattern not found in USER_INPUT_PATTERNS") + + +FALSE_POSITIVES = [ + "res.setCookie(token)", + "PQsendQuery(conn, sql)", + "req.getHeader('X')", + "parseMultipartFile(x)", + "renderBody(html)", + "buildForm(fields)", +] + +TRUE_POSITIVES = [ + "def handler(c: str = Cookie(None)): ...", + "def handler(q: str = Query(...)): ...", + "def handler(b: Item = Body(...)): ...", + "def handler(h: str = Header(None)): ...", +] + + +def test_input_pattern_rejects_substring_false_positives(): + pat = _fastapi_pattern() + for code in FALSE_POSITIVES: + assert pat.search(code) is None, f"false positive: {code!r} matched {pat.pattern!r}" + + +def test_input_pattern_still_matches_standalone_symbols(): + pat = _fastapi_pattern() + for code in TRUE_POSITIVES: + assert pat.search(code) is not None, f"regressed: {code!r} no longer matches" diff --git a/libs/openant-core/tests/test_library_mode_reachability.py b/libs/openant-core/tests/test_library_mode_reachability.py new file mode 100644 index 00000000..6db8a160 --- /dev/null +++ b/libs/openant-core/tests/test_library_mode_reachability.py @@ -0,0 +1,122 @@ +"""Library-mode reachability seeding (BUG-005). + +A pure library exposes no main/route/CLI entry point, so the structural detector +finds nothing and `apply_reachability_filter` drops EVERY unit — the library +(including any vulnerable sink it contains) is never analysed. Library-mode seeds +the public API surface so the forward BFS pulls in the rest. + +These tests pin: (1) the mode-OFF baseline, (2) the public API becomes +reachable when ON (and its private callee comes along via the call edge), (3) a +truly-unreferenced private function stays out, and — adversarially — (4) turning +the mode ON for an APP can only ADD reachable units, never remove one (union-only +seed merge), so existing app scans are never degraded. + +NOTE: stacked on PR #75. On master a no-entry-point library blacks out (0 units), +which is the bug this PR fixes. PR #75's zero-seed fallback already prevents that +blackout — bluntly — by returning ALL units unfiltered when no entry point is +detected. So the mode-OFF baseline here is "all units unfiltered" (#75), and +library-mode ON refines it to the precise public-API-reachable subset. +""" + +import json +import sys +from pathlib import Path + +_CORE_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(_CORE_ROOT)) + +from core.parser_adapter import apply_reachability_filter + + +def _run(tmp_path, functions, call_graph, *, library_mode, entry_types=None): + """Write a call_graph.json + dataset and run the filter; return kept unit ids.""" + entry_types = entry_types or {} + reverse = {} + for caller, callees in call_graph.items(): + for callee in callees: + reverse.setdefault(callee, []).append(caller) + # functions carry name (+ optional unit_type to trip the structural detector) + fns = {fid: {"name": fid.split(":")[-1].split(".")[-1], + "unit_type": entry_types.get(fid, "function")} for fid in functions} + (tmp_path / "call_graph.json").write_text(json.dumps( + {"functions": fns, "call_graph": call_graph, "reverse_call_graph": reverse})) + dataset = {"units": [{"id": fid, "unit_type": entry_types.get(fid, "function")} + for fid in functions]} + out = apply_reachability_filter(dataset, str(tmp_path), "reachable", + library_mode=library_mode) + return {u["id"] for u in out["units"]} + + +# library: public_api() -> _sink() (no structural entry point) +_LIB_FNS = ["lib.py:public_api", "lib.py:_sink"] +_LIB_CG = {"lib.py:public_api": ["lib.py:_sink"]} + + +def test_library_mode_off_returns_all_unfiltered(tmp_path): + """Mode off (stacked on #75): a no-entry-point library is NOT blacked out — + #75's zero-seed fallback returns all units unfiltered. Library-mode ON refines + this to the public-API-reachable subset (see precision test below).""" + kept = _run(tmp_path, _LIB_FNS, _LIB_CG, library_mode=False) + assert kept == set(_LIB_FNS), f"expected #75 all-unfiltered fallback, got {kept}" + + +def test_library_public_api_reachable_when_mode_on(tmp_path): + """Mode on: the public API is seeded, and its private callee comes along.""" + kept = _run(tmp_path, _LIB_FNS, _LIB_CG, library_mode=True) + assert "lib.py:public_api" in kept, f"public API not seeded: {kept}" + assert "lib.py:_sink" in kept, f"private callee of the public API not reached: {kept}" + + +def test_unreferenced_private_stays_out(tmp_path): + """Precision: a private function nothing calls is NOT seeded (only the public + surface is) — so library-mode doesn't blanket-seed every unit.""" + fns = _LIB_FNS + ["lib.py:_orphan"] + kept = _run(tmp_path, fns, _LIB_CG, library_mode=True) + assert "lib.py:_orphan" not in kept, f"unreferenced private wrongly seeded: {kept}" + + +# app: main() is a route_handler entry; helper() is its callee; _dead() is unreferenced +_APP_FNS = ["app.py:main", "app.py:helper", "app.py:_dead"] +_APP_CG = {"app.py:main": ["app.py:helper"]} +_APP_ENTRY = {"app.py:main": "route_handler"} + + +def test_app_baseline_mode_off(tmp_path): + """App with a real entry point: normal reachable set when mode off.""" + kept = _run(tmp_path, _APP_FNS, _APP_CG, library_mode=False, entry_types=_APP_ENTRY) + assert kept == {"app.py:main", "app.py:helper"}, f"app baseline changed: {kept}" + + +def test_app_mode_on_is_additive_only(tmp_path): + """Adversarial: turning library-mode ON for an app can only ADD reachable units + (union-only seed merge) — it must never drop one the app scan already had.""" + off = _run(tmp_path, _APP_FNS, _APP_CG, library_mode=False, entry_types=_APP_ENTRY) + on = _run(tmp_path, _APP_FNS, _APP_CG, library_mode=True, entry_types=_APP_ENTRY) + assert off <= on, f"library-mode REMOVED app units: off={off} on={on}" + assert off == {"app.py:main", "app.py:helper"} + + +def test_parse_repository_wiring(tmp_path): + """Integration guard: library_mode must flow parse_repository -> _parse_python -> + apply_reachability_filter. (A unit test on the filter alone missed a wiring bug + where `_parse_python` referenced library_mode before it was threaded.)""" + from core.parser_adapter import parse_repository + repo = tmp_path / "repo"; repo.mkdir() + (repo / "lib.py").write_text( + "def public_api(x):\n return _sink(x)\n\ndef _sink(x):\n return eval(x)\n") + import json as _json + + def _kept(library_mode): + out = tmp_path / f"out_{library_mode}"; out.mkdir() + parse_repository(repo_path=str(repo), output_dir=str(out), language="python", + processing_level="reachable", library_mode=library_mode) + ds = _json.loads((out / "dataset.json").read_text()) + return {u.get("id") for u in ds.get("units", [])} + + # Stacked on #75: mode off returns all units unfiltered (zero-seed fallback), + # not a blackout. Mode on refines to the public-API-reachable subset. + assert _kept(False) == {"lib.py:public_api", "lib.py:_sink"}, \ + "mode off: expected #75 all-unfiltered fallback" + on = _kept(True) + assert any(i.endswith(":public_api") for i in on), f"public api not analysed: {on}" + assert any(i.endswith(":_sink") for i in on), f"eval sink not analysed: {on}" diff --git a/libs/openant-core/tests/test_library_seed_ids.py b/libs/openant-core/tests/test_library_seed_ids.py new file mode 100644 index 00000000..e77071ff --- /dev/null +++ b/libs/openant-core/tests/test_library_seed_ids.py @@ -0,0 +1,63 @@ +"""Fix C — shared library_seed_ids: public-API seed set, both key casings. + +The subprocess pipelines normalize function records to camelCase (`isExported`) +while the on-disk call_graph and the Python path use snake_case (`is_exported`). +`library_seed_ids` must seed the exported, non-name-private functions under EITHER +casing, defaulting to exported when neither field is present (over-seed, never +under-seed). This is what lets a C/JS/etc. library's public API surface seed the +reachability BFS instead of blacking out. +""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # libs/openant-core + +from utilities.agentic_enhancer import library_seed_ids # noqa: E402 + + +def test_snake_case_exported_seeded(): + fns = {"f.c:pub": {"name": "pub", "is_exported": True}} + assert library_seed_ids(fns) == {"f.c:pub"} + + +def test_snake_case_static_not_seeded(): + fns = {"f.c:priv": {"name": "priv", "is_exported": False}} + assert library_seed_ids(fns) == set() + + +def test_camel_case_exported_seeded(): + # JS/normalized-pipeline shape. + fns = {"f.js:pub": {"name": "pub", "isExported": True}} + assert library_seed_ids(fns) == {"f.js:pub"} + + +def test_camel_case_unexported_not_seeded(): + fns = {"f.js:priv": {"name": "priv", "isExported": False}} + assert library_seed_ids(fns) == set() + + +def test_missing_field_defaults_exported(): + # Parsers without an export field (python/ruby/php) default to exported. + fns = {"m.py:helper": {"name": "helper"}} + assert library_seed_ids(fns) == {"m.py:helper"} + + +def test_leading_underscore_name_is_private(): + fns = { + "m.py:_internal": {"name": "_internal"}, + "m.py:public": {"name": "public"}, + } + assert library_seed_ids(fns) == {"m.py:public"} + + +def test_name_falls_back_to_func_id_tail(): + # No 'name' field -> derive from the func_id tail, strip a dotted qualifier. + fns = {"f.c:Mod.run": {"is_exported": True}} + assert library_seed_ids(fns) == {"f.c:Mod.run"} + + +def test_exported_but_underscore_still_excluded(): + # Name-private wins even when exported (a public-but-_-prefixed symbol is + # conventionally internal); over-seeding bias does not override the name rule. + fns = {"f.c:_x": {"name": "_x", "is_exported": True}} + assert library_seed_ids(fns) == set() diff --git a/libs/openant-core/tests/test_parse_fresh.py b/libs/openant-core/tests/test_parse_fresh.py index 93f04d3c..afc05970 100644 --- a/libs/openant-core/tests/test_parse_fresh.py +++ b/libs/openant-core/tests/test_parse_fresh.py @@ -21,7 +21,7 @@ def _make_stub_parser(record): time it is invoked, then writes a fresh dataset itself so the rest of `parse_repository` has something to work with. """ - def _stub(repo_path, output_dir, processing_level, skip_tests=True, name=None): + def _stub(repo_path, output_dir, processing_level, skip_tests=True, name=None, library_mode=False): dataset_path = os.path.join(output_dir, "dataset.json") record["dataset_existed_when_parser_ran"] = os.path.exists(dataset_path) # Mimic real parser output diff --git a/libs/openant-core/tests/test_php_trait_self_call.py b/libs/openant-core/tests/test_php_trait_self_call.py new file mode 100644 index 00000000..dc070d2e --- /dev/null +++ b/libs/openant-core/tests/test_php_trait_self_call.py @@ -0,0 +1,101 @@ +"""Regression test for F12 sub-defect (3): trait-composition self-calls. + +Sub-defects (1) [the `trait composition index: a class that pulls a method in via +`use TraitName;` had no edge from `$this->m()` / `self::m()` to the trait's +method, because `_resolve_self_call` only looked at methods physically declared +in the class body and never the methods of its used traits. + +Two layers are exercised: + * builder layer -- given a `traits` field on the class record, the + CallGraphBuilder must fall back to the used traits' methods. + * extractor layer -- the FunctionExtractor must populate that `traits` field + from the in-class `use_declaration` node so the builder has data to use. + +Loads both modules under UNIQUE importlib names (call_graph_builder / +function_extractor are basenames shared by every parser). +""" +import importlib.util +import sys +from pathlib import Path + +CORE = Path(__file__).resolve().parents[1] +if str(CORE) not in sys.path: + sys.path.insert(0, str(CORE)) + + +def _load(unique, relpath): + spec = importlib.util.spec_from_file_location(unique, str(CORE / relpath)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +_cgb = _load("php_call_graph_builder_trait", "parsers/php/call_graph_builder.py") +_fe = _load("php_function_extractor_trait", "parsers/php/function_extractor.py") +CallGraphBuilder = _cgb.CallGraphBuilder +FunctionExtractor = _fe.FunctionExtractor + + +def _build(funcs, classes=None, imports=None): + b = CallGraphBuilder({"functions": funcs, "classes": classes or {}, "imports": imports or {}, + "repository": "/r"}) + b.build_call_graph() + return b + + +# F12 #3 builder layer: $this->g() and self::g() resolve into a used trait's method. +def test_trait_self_call_resolves_into_used_trait(): + b = _build({ + "a.php:C.f": {"name": "f", "file_path": "a.php", "class_name": "C", + "code": "function f() { $this->g(); }"}, + "a.php:C.h": {"name": "h", "file_path": "a.php", "class_name": "C", + "code": "function h() { self::g(); }"}, + "a.php:T.g": {"name": "g", "file_path": "a.php", "class_name": "T", + "code": "function g() {}"}, + }, classes={ + "a.php:C": {"name": "C", "file_path": "a.php", "superclass": None, "traits": ["T"]}, + "a.php:T": {"name": "T", "file_path": "a.php", "superclass": None, "traits": []}, + }) + assert b.call_graph.get("a.php:C.f") == ["a.php:T.g"], b.call_graph + assert b.call_graph.get("a.php:C.h") == ["a.php:T.g"], b.call_graph + + +# F12 #3 extractor layer: the in-class `use T;` is captured onto the class record, +# and the full extract->build pipeline yields the trait edges from real source. +def test_extractor_captures_trait_use_and_builds_edges(tmp_path): + src = ( + "g(); }\n" + " function h() { self::g(); }\n" + "}\n" + ) + f = tmp_path / "trait_case.php" + f.write_text(src) + + ex = FunctionExtractor(str(tmp_path)) + out = ex.extract_all() + + # Class record must carry the used trait. + c_key = next(k for k in out["classes"] if k.endswith(":C")) + assert "T" in out["classes"][c_key].get("traits", []), out["classes"][c_key] + + b = CallGraphBuilder(out) + b.build_call_graph() + + f_id = next(k for k in out["functions"] + if out["functions"][k].get("name") == "f" + and out["functions"][k].get("class_name") == "C") + h_id = next(k for k in out["functions"] + if out["functions"][k].get("name") == "h" + and out["functions"][k].get("class_name") == "C") + g_id = next(k for k in out["functions"] + if out["functions"][k].get("name") == "g" + and out["functions"][k].get("class_name") == "T") + + assert g_id in b.call_graph.get(f_id, []), b.call_graph.get(f_id) + assert g_id in b.call_graph.get(h_id, []), b.call_graph.get(h_id) diff --git a/libs/openant-core/tests/test_threat_model_untrusted_input_gate.py b/libs/openant-core/tests/test_threat_model_untrusted_input_gate.py new file mode 100644 index 00000000..8d55def4 --- /dev/null +++ b/libs/openant-core/tests/test_threat_model_untrusted_input_gate.py @@ -0,0 +1,83 @@ +"""Fix A — threat-model gate: don't suppress untrusted-input bugs for data libraries. + +`format_app_context_for_prompt` (Stage 1) and `format_app_context_for_verification` +(Stage 2) historically emitted a "this is a CLI tool/library, only flag REMOTE +attackers, not local users" suppression block whenever `requires_remote_trigger` +was False — which the generator sets for EVERY library. That discards exactly the +bug class of a parser/deserializer/codec, whose untrusted INPUT DATA is the attack +surface even with no network listener (the tree-sitter case). + +Fix: gate the suppression on `ApplicationContext.suppress_local_only()`, which keeps +the clause only when NO trust boundary is `untrusted`. A library that ingests +untrusted data (`source_code_being_parsed: untrusted`) is therefore analysed, while +a genuine no-attack-surface library (all-trusted) is unchanged — no new field, reuses +the already-captured `trust_boundaries`. +""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # libs/openant-core + +from context.application_context import ApplicationContext # noqa: E402 +from prompts.vulnerability_analysis import format_app_context_for_prompt # noqa: E402 +from prompts.verification_prompts import format_app_context_for_verification # noqa: E402 + +# Sentinels that appear ONLY inside the suppression block of each formatter. +_STAGE1_SUPPRESSION = "have local access" # vulnerability_analysis.py +_STAGE2_SUPPRESSION = "local filesystem access" # verification_prompts.py + + +def _lib(trust_boundaries, *, remote=False): + return ApplicationContext( + application_type="library", + purpose="x", + trust_boundaries=trust_boundaries, + requires_remote_trigger=remote, + ) + + +# --- the method itself -------------------------------------------------------- +def test_suppress_when_all_trusted(): + assert _lib({"cli_args": "trusted", "config": "trusted"}).suppress_local_only() is True + + +def test_do_not_suppress_when_any_untrusted(): + assert _lib({"source_code_being_parsed": "untrusted", "config": "trusted"}).suppress_local_only() is False + + +def test_untrusted_match_is_case_insensitive(): + # trust_boundaries values are LLM-generated; tolerate case deviation. + assert _lib({"input": "Untrusted"}).suppress_local_only() is False + assert _lib({"input": "UNTRUSTED"}).suppress_local_only() is False + + +def test_do_not_suppress_when_remote(): + # web_app-style: remote trigger always means no local-only suppression. + assert _lib({"cli_args": "trusted"}, remote=True).suppress_local_only() is False + + +def test_empty_boundaries_still_suppress(): + # No declared boundaries + local-only library -> keep the conservative suppression. + assert _lib({}).suppress_local_only() is True + + +# --- Stage 1 formatter -------------------------------------------------------- +def test_stage1_all_trusted_keeps_suppression(): + out = format_app_context_for_prompt(_lib({"cli_args": "trusted"})) + assert _STAGE1_SUPPRESSION in out + + +def test_stage1_untrusted_input_drops_suppression(): + out = format_app_context_for_prompt(_lib({"source_code_being_parsed": "untrusted"})) + assert _STAGE1_SUPPRESSION not in out + + +# --- Stage 2 formatter -------------------------------------------------------- +def test_stage2_all_trusted_keeps_suppression(): + out = format_app_context_for_verification(_lib({"cli_args": "trusted"})) + assert _STAGE2_SUPPRESSION in out + + +def test_stage2_untrusted_input_drops_suppression(): + out = format_app_context_for_verification(_lib({"source_code_being_parsed": "untrusted"})) + assert _STAGE2_SUPPRESSION not in out diff --git a/libs/openant-core/utilities/agentic_enhancer/__init__.py b/libs/openant-core/utilities/agentic_enhancer/__init__.py index 39349261..a13756e7 100644 --- a/libs/openant-core/utilities/agentic_enhancer/__init__.py +++ b/libs/openant-core/utilities/agentic_enhancer/__init__.py @@ -20,7 +20,7 @@ ) from .repository_index import RepositoryIndex, load_index_from_file from .tools import TOOL_DEFINITIONS, ToolExecutor -from .entry_point_detector import EntryPointDetector +from .entry_point_detector import EntryPointDetector, blackout_warning, library_seed_ids from .reachability_analyzer import ReachabilityAnalyzer __all__ = [ @@ -33,5 +33,7 @@ "TOOL_DEFINITIONS", "ToolExecutor", "EntryPointDetector", + "blackout_warning", + "library_seed_ids", "ReachabilityAnalyzer" ] diff --git a/libs/openant-core/utilities/agentic_enhancer/entry_point_detector.py b/libs/openant-core/utilities/agentic_enhancer/entry_point_detector.py index 5b278c56..edb8da57 100644 --- a/libs/openant-core/utilities/agentic_enhancer/entry_point_detector.py +++ b/libs/openant-core/utilities/agentic_enhancer/entry_point_detector.py @@ -84,7 +84,7 @@ def _unit_type(func_data: Dict) -> str: r'request\.environ', # FastAPI r'request\.(query_params|body|json)', - r'(Query|Body|Form|File|Header|Cookie)\s*\(', + r'\b(Query|Body|Form|File|Header|Cookie)\s*\(', # Django r'request\.(GET|POST|data|FILES|body)', r'self\.request\.(GET|POST|data)', @@ -94,7 +94,7 @@ def _unit_type(func_data: Dict) -> str: # CLI arguments r'sys\.argv', r'argparse\.', - r'ArgumentParser\s*\(', + r'\bArgumentParser\s*\(', r'click\.(argument|option)', # Standard input r'\binput\s*\(', @@ -213,6 +213,14 @@ def _get_entry_point_reasons(self, func_data: Dict) -> List[str]: if unit_type in ENTRY_POINT_TYPES: reasons.append(f'unit_type:{unit_type}') + # Check 1b: A function named `main` is a program execution root by name, + # even when the extractor classified its unit_type as something else + # (defensive: covers language extractors that emit a generic unit_type + # for main). A program's main is an entry point; over-approximating it + # is reachability-safe. + elif func_data.get('name') == 'main': + reasons.append('name:main') + # Check 2: Decorators indicate entry point decorators = func_data.get('decorators', []) decorators_str = ' '.join(decorators) @@ -276,3 +284,72 @@ def get_statistics(self) -> Dict: 'by_unit_type': by_type, 'by_reason_category': by_reason, } + + +def library_seed_ids(functions): + """Public-API seed set for library-mode reachability. + + A pure library exposes no main/route/CLI entry point, so the structural + detector finds nothing and the whole library is filtered out (0 reachable). + In library-mode the *public surface* IS the entry surface: seed every + exported/public function and let the forward BFS pull in its callees. + + Public = exported AND not name-private. Honours ``is_exported``/``isExported`` + when the parser provides it (C/Go/JS exclude static/unexported); for parsers + without the field (python/ruby/php) it defaults True and the leading-underscore + name heuristic decides. Both key casings are accepted because the subprocess + pipelines normalize to camelCase while the on-disk call_graph is snake_case. + The bias is intentionally toward over-seeding (more reachable = more analysed), + never under-seeding. + """ + seeds = set() + for func_id, fd in functions.items(): + name = (fd.get("name") or func_id.rsplit(":", 1)[-1]).split(".")[-1] + exported = fd.get("is_exported", fd.get("isExported", True)) + if exported and not name.startswith("_"): + seeds.add(func_id) + return seeds + + +# Reason categories that indicate a STRUCTURAL entry point — a real route, program +# main, CLI command, framework handler, or decorator-marked endpoint — as opposed +# to an INCIDENTAL match (code merely contains an input-reading pattern). A result +# seeded ONLY by incidental matches is the library-blackout signature: the public +# API was never a seed, so the BFS dropped the core. +_STRUCTURAL_REASON_CATEGORIES = {"unit_type", "decorator", "name"} + + +def blackout_warning(entry_point_details, original_count, reachable_count, + library_mode=False, reduction_threshold=0.90): + """Advisory string when a reachability result looks like a silent library + blackout, else None. This is ADVISORY ONLY — it never changes which units + are kept. + + Two triggers (both off when ``library_mode`` is set, since then the public + API was deliberately seeded and a high reduction is the intended result): + * total blackout — 0 of N units kept (no seedable frontier); or + * partial blackout — >= ``reduction_threshold`` pruned AND no STRUCTURAL + entry point was found (every seed is an incidental ``input_pattern`` + match). This is the case that slips past the zero-seed net: a handful of + incidental seeds yield a 96%+ reduction that looks like success while the + real public API surface was never analysed (e.g. a C/JS parser library). + """ + if original_count <= 0 or library_mode: + return None + if reachable_count == 0: + return (f"Reachability kept 0 of {original_count} units — total blackout " + f"(no entry point could seed the frontier). If this is a library, " + f"re-run with --library-mode to seed the exported public API surface.") + reduction = 1.0 - (reachable_count / original_count) + structural = sum( + 1 for d in (entry_point_details or {}).values() + if any(r.split(":", 1)[0] in _STRUCTURAL_REASON_CATEGORIES + for r in d.get("reasons", [])) + ) + if reduction >= reduction_threshold and structural == 0: + return (f"Reachability kept {reachable_count} of {original_count} units " + f"({reduction * 100:.0f}% pruned) but found NO structural entry point " + f"(route/main/CLI/handler) — only incidental code-pattern seeds. This is " + f"the library-blackout pattern: the public API was not seeded, so the core " + f"was dropped. Re-run with --library-mode to seed the exported public API.") + return None