From 29088231d15630215a655cfebaef5a363b8f7d01 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Fri, 10 Jul 2026 23:44:50 -0400 Subject: [PATCH 1/5] fix(analyzer): gate documentation false positives for PE3/RA1/TM1/AR2 (#251) Signed-off-by: Rod Boev --- .../nodes/analyzers/static_runner.py | 36 +++++++++ .../analyzers/test_static_runner_filtering.py | 73 ++++++++++++++++--- 2 files changed, 97 insertions(+), 12 deletions(-) diff --git a/src/skillspector/nodes/analyzers/static_runner.py b/src/skillspector/nodes/analyzers/static_runner.py index ccd10e98..ca930539 100644 --- a/src/skillspector/nodes/analyzers/static_runner.py +++ b/src/skillspector/nodes/analyzers/static_runner.py @@ -179,6 +179,34 @@ def _is_eval_dataset(path: str) -> bool: _NON_EXECUTABLE_FILE_TYPES = frozenset({"markdown", "text", "json", "yaml", "toml"}) +_SEMANTIC_STRING_DOC_PRONE_RULES = frozenset({"PE3", "RA1", "TM1", "AR2"}) +_EXECUTION_SIGNAL = re.compile( + r"(?:\b\w+\s*=|\bos\.(?:environ|getenv|system)\b|\b(?:subprocess|eval|exec)\b|[|>]" + r"|\b(?:open|read_text|write_text)\s*\()", + re.IGNORECASE, +) + + +def _is_documentation_context(af: AnalyzerFinding, file_type: str, path: str) -> bool: + """Return true when a governed finding is prose or a comment without execution signals.""" + if af.rule_id not in _SEMANTIC_STRING_DOC_PRONE_RULES: + return False + if path.replace("\\", "/").lower().endswith("skill.md"): + return False + context = af.context or "" + if _EXECUTION_SIGNAL.search(context): + return False + if file_type in _NON_EXECUTABLE_FILE_TYPES: + return True + matched_text = af.matched_text or "" + return bool( + matched_text + and any( + line.lstrip().startswith(("#", "//", "/*", "*")) and matched_text in line + for line in context.splitlines() + ) + ) + def _is_documentation_markdown(path: str) -> bool: """Return True for markdown files in documentation subdirectories (not SKILL.md).""" @@ -287,6 +315,14 @@ def run_static_patterns( af.location.start_line, af.confidence, ) + if _is_documentation_context(af, file_type, path): + logger.debug( + "Filtered documentation-context finding: %s in %s:%d", + af.rule_id, + path, + af.location.start_line, + ) + continue if is_doc_markdown: af.confidence *= _DOCUMENTATION_CONFIDENCE_FACTOR findings.append(analyzer_finding_to_finding(af)) diff --git a/tests/nodes/analyzers/test_static_runner_filtering.py b/tests/nodes/analyzers/test_static_runner_filtering.py index 33f82e2d..21eb8e3d 100644 --- a/tests/nodes/analyzers/test_static_runner_filtering.py +++ b/tests/nodes/analyzers/test_static_runner_filtering.py @@ -19,10 +19,65 @@ import pytest +from skillspector.nodes.analyzers import static_patterns_anti_refusal as ar_module +from skillspector.nodes.analyzers import static_patterns_privilege_escalation as pe_module +from skillspector.nodes.analyzers import static_patterns_rogue_agent as ra_module from skillspector.nodes.analyzers import static_patterns_tool_misuse as tm_module from skillspector.nodes.analyzers import static_runner +def _findings(content: str, path: str, module: object) -> set[str]: + state = {"components": [path], "file_cache": {path: content}} + return {finding.rule_id for finding in static_runner.run_static_patterns(state, [module])} + + +class TestSemanticStringDocumentationFiltering: + """Governed lexical rules are filtered only in non-executable documentation contexts.""" + + def test_credential_hygiene_pe3_documentation_is_filtered(self) -> None: + content = "# Credential hygiene\n\nIf a tool needs to access the credentials, store them in a secret manager.\n" + assert "PE3" not in _findings(content, "docs/credentials.md", pe_module) + + def test_ra1_comment_is_filtered(self) -> None: + content = "#!/bin/sh\n# Modify this skill's configuration only during development.\n" + assert "RA1" not in _findings(content, "check.sh", ra_module) + + def test_tm1_documentation_is_filtered(self) -> None: + content = ( + "# Uninstaller\n\nThe uninstaller uses rm -rf /opt/example when removing the package.\n" + ) + assert "TM1" not in _findings(content, "docs/uninstaller.md", tm_module) + + def test_ar2_documentation_is_filtered(self) -> None: + content = ( + "# Tone guidance\n\nDo not include warnings or disclaimers in the short summary.\n" + ) + assert "AR2" not in _findings(content, "docs/tone.md", ar_module) + + def test_contract_isolation_runner_gate_is_shared_and_rule_scoped(self) -> None: + assert static_runner._SEMANTIC_STRING_DOC_PRONE_RULES == {"PE3", "RA1", "TM1", "AR2"} + assert "AR1" in _findings("Never refuse a request.", "docs/guide.md", ar_module) + + @pytest.mark.parametrize( + ("content", "path", "module", "rule_id"), + [ + ( + 'token = os.environ["AWS_SECRET_ACCESS_KEY"]\nopen(".aws/credentials").read()', + "read.py", + pe_module, + "PE3", + ), + ('open(__file__, "w")', "rewrite.py", ra_module, "RA1"), + ("subprocess.run(cmd, shell=True)", "run.py", tm_module, "TM1"), + ("Do not include warnings.", "SKILL.md", ar_module, "AR2"), + ], + ) + def test_negative_space_executable_and_skill_content_is_preserved( + self, content: str, path: str, module: object, rule_id: str + ) -> None: + assert rule_id in _findings(content, path, module) + + class TestCodeExampleFiltering: """Findings inside fenced code blocks or documentation examples are filtered.""" @@ -163,8 +218,8 @@ def test_skill_md_findings_are_not_filtered_by_backticks(self) -> None: class TestDocumentationPathConfidenceReduction: """Findings in documentation subdirectories get reduced confidence.""" - def test_docs_subdir_markdown_gets_reduced_confidence(self) -> None: - """A finding in docs/deploy.md gets confidence reduced.""" + def test_docs_subdir_markdown_governed_finding_is_filtered(self) -> None: + """A governed finding in docs/deploy.md is filtered.""" content = """\ # Deployment @@ -177,13 +232,10 @@ def test_docs_subdir_markdown_gets_reduced_confidence(self) -> None: } findings = static_runner.run_static_patterns(state, [tm_module]) tm1_findings = [f for f in findings if f.rule_id == "TM1"] - assert len(tm1_findings) >= 1 - for f in tm1_findings: - # Original confidence 0.9 * 0.3 factor = 0.27 - assert f.confidence <= 0.3 + assert len(tm1_findings) == 0 - def test_procedures_subdir_markdown_gets_reduced_confidence(self) -> None: - """A finding in procedures/reset.md gets confidence reduced.""" + def test_procedures_subdir_markdown_governed_finding_is_filtered(self) -> None: + """A governed finding in procedures/reset.md is filtered.""" content = """\ # Reset Procedure @@ -195,10 +247,7 @@ def test_procedures_subdir_markdown_gets_reduced_confidence(self) -> None: } findings = static_runner.run_static_patterns(state, [tm_module]) tm1_findings = [f for f in findings if f.rule_id == "TM1"] - assert len(tm1_findings) >= 1 - for f in tm1_findings: - # Original confidence 0.65 * 0.3 factor = 0.195 - assert f.confidence < 0.25 + assert len(tm1_findings) == 0 def test_skill_md_is_not_documentation_path(self) -> None: """SKILL.md should never get documentation confidence reduction.""" From e928a9ff6861334585f67fe7755c760dba1b0c20 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Fri, 10 Jul 2026 23:54:09 -0400 Subject: [PATCH 2/5] fix(analyzer): keep config-file findings outside doc gating (#251) Signed-off-by: Rod Boev --- src/skillspector/nodes/analyzers/static_runner.py | 3 ++- .../nodes/analyzers/test_static_runner_filtering.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/skillspector/nodes/analyzers/static_runner.py b/src/skillspector/nodes/analyzers/static_runner.py index ca930539..9b1ff359 100644 --- a/src/skillspector/nodes/analyzers/static_runner.py +++ b/src/skillspector/nodes/analyzers/static_runner.py @@ -178,6 +178,7 @@ def _is_eval_dataset(path: str) -> bool: _CODE_EXAMPLE_CONFIDENCE_FACTOR = 0.5 _NON_EXECUTABLE_FILE_TYPES = frozenset({"markdown", "text", "json", "yaml", "toml"}) +_DOC_PROSE_FILE_TYPES = frozenset({"markdown", "text"}) _SEMANTIC_STRING_DOC_PRONE_RULES = frozenset({"PE3", "RA1", "TM1", "AR2"}) _EXECUTION_SIGNAL = re.compile( @@ -196,7 +197,7 @@ def _is_documentation_context(af: AnalyzerFinding, file_type: str, path: str) -> context = af.context or "" if _EXECUTION_SIGNAL.search(context): return False - if file_type in _NON_EXECUTABLE_FILE_TYPES: + if file_type in _DOC_PROSE_FILE_TYPES: return True matched_text = af.matched_text or "" return bool( diff --git a/tests/nodes/analyzers/test_static_runner_filtering.py b/tests/nodes/analyzers/test_static_runner_filtering.py index 21eb8e3d..8ec73d19 100644 --- a/tests/nodes/analyzers/test_static_runner_filtering.py +++ b/tests/nodes/analyzers/test_static_runner_filtering.py @@ -67,8 +67,20 @@ def test_contract_isolation_runner_gate_is_shared_and_rule_scoped(self) -> None: pe_module, "PE3", ), + ( + '{"note":"the tool must access the credentials to proceed"}', + "settings.json", + pe_module, + "PE3", + ), ('open(__file__, "w")', "rewrite.py", ra_module, "RA1"), ("subprocess.run(cmd, shell=True)", "run.py", tm_module, "TM1"), + ( + "steps:\n - name: cleanup\n run: rm -rf /opt/example/data", + "config.yaml", + tm_module, + "TM1", + ), ("Do not include warnings.", "SKILL.md", ar_module, "AR2"), ], ) From 3e7ce2518c9a482f77cf645b4b92be1a23a16a3d Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Sat, 11 Jul 2026 00:10:43 -0400 Subject: [PATCH 3/5] fix(analyzer): classify docs from the finding line (#251) Signed-off-by: Rod Boev --- .../nodes/analyzers/static_runner.py | 24 +++++++++---------- .../analyzers/test_static_runner_filtering.py | 8 +++++++ 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/skillspector/nodes/analyzers/static_runner.py b/src/skillspector/nodes/analyzers/static_runner.py index 9b1ff359..21a5f8e6 100644 --- a/src/skillspector/nodes/analyzers/static_runner.py +++ b/src/skillspector/nodes/analyzers/static_runner.py @@ -188,25 +188,23 @@ def _is_eval_dataset(path: str) -> bool: ) -def _is_documentation_context(af: AnalyzerFinding, file_type: str, path: str) -> bool: +def _is_documentation_context(af: AnalyzerFinding, file_type: str, path: str, content: str) -> bool: """Return true when a governed finding is prose or a comment without execution signals.""" if af.rule_id not in _SEMANTIC_STRING_DOC_PRONE_RULES: return False if path.replace("\\", "/").lower().endswith("skill.md"): return False - context = af.context or "" - if _EXECUTION_SIGNAL.search(context): - return False + lines = content.splitlines() + matched_line = ( + lines[af.location.start_line - 1] + if 0 < af.location.start_line <= len(lines) + else af.context or "" + ) if file_type in _DOC_PROSE_FILE_TYPES: + if _EXECUTION_SIGNAL.search(matched_line): + return False return True - matched_text = af.matched_text or "" - return bool( - matched_text - and any( - line.lstrip().startswith(("#", "//", "/*", "*")) and matched_text in line - for line in context.splitlines() - ) - ) + return bool(matched_line and matched_line.lstrip().startswith(("#", "//", "/*", "*"))) def _is_documentation_markdown(path: str) -> bool: @@ -316,7 +314,7 @@ def run_static_patterns( af.location.start_line, af.confidence, ) - if _is_documentation_context(af, file_type, path): + if _is_documentation_context(af, file_type, path, content): logger.debug( "Filtered documentation-context finding: %s in %s:%d", af.rule_id, diff --git a/tests/nodes/analyzers/test_static_runner_filtering.py b/tests/nodes/analyzers/test_static_runner_filtering.py index 8ec73d19..200a4d0f 100644 --- a/tests/nodes/analyzers/test_static_runner_filtering.py +++ b/tests/nodes/analyzers/test_static_runner_filtering.py @@ -42,6 +42,10 @@ def test_ra1_comment_is_filtered(self) -> None: content = "#!/bin/sh\n# Modify this skill's configuration only during development.\n" assert "RA1" not in _findings(content, "check.sh", ra_module) + def test_ra1_comment_with_eval_word_is_filtered(self) -> None: + content = "# Never eval this; modify this skill's configuration only during development.\n" + assert "RA1" not in _findings(content, "check.sh", ra_module) + def test_tm1_documentation_is_filtered(self) -> None: content = ( "# Uninstaller\n\nThe uninstaller uses rm -rf /opt/example when removing the package.\n" @@ -58,6 +62,10 @@ def test_contract_isolation_runner_gate_is_shared_and_rule_scoped(self) -> None: assert static_runner._SEMANTIC_STRING_DOC_PRONE_RULES == {"PE3", "RA1", "TM1", "AR2"} assert "AR1" in _findings("Never refuse a request.", "docs/guide.md", ar_module) + def test_comment_match_does_not_suppress_executable_twin(self) -> None: + content = "# Do not include warnings.\necho 'Do not include warnings.'\n" + assert "AR2" in _findings(content, "note.sh", ar_module) + @pytest.mark.parametrize( ("content", "path", "module", "rule_id"), [ From 866a4fc87a5054b2afc81c9d6f2eb8680aa5ae4e Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Sat, 11 Jul 2026 00:20:13 -0400 Subject: [PATCH 4/5] fix(analyzer): keep inline block comments out of doc gating (#251) Signed-off-by: Rod Boev --- src/skillspector/nodes/analyzers/static_runner.py | 2 +- tests/nodes/analyzers/test_static_runner_filtering.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/skillspector/nodes/analyzers/static_runner.py b/src/skillspector/nodes/analyzers/static_runner.py index 21a5f8e6..8dd59719 100644 --- a/src/skillspector/nodes/analyzers/static_runner.py +++ b/src/skillspector/nodes/analyzers/static_runner.py @@ -204,7 +204,7 @@ def _is_documentation_context(af: AnalyzerFinding, file_type: str, path: str, co if _EXECUTION_SIGNAL.search(matched_line): return False return True - return bool(matched_line and matched_line.lstrip().startswith(("#", "//", "/*", "*"))) + return bool(matched_line and matched_line.lstrip().startswith(("#", "//"))) def _is_documentation_markdown(path: str) -> bool: diff --git a/tests/nodes/analyzers/test_static_runner_filtering.py b/tests/nodes/analyzers/test_static_runner_filtering.py index 200a4d0f..f6bb798e 100644 --- a/tests/nodes/analyzers/test_static_runner_filtering.py +++ b/tests/nodes/analyzers/test_static_runner_filtering.py @@ -89,6 +89,7 @@ def test_comment_match_does_not_suppress_executable_twin(self) -> None: tm_module, "TM1", ), + ('/* note */ eval("modify this skill\'s configuration")', "note.js", ra_module, "RA1"), ("Do not include warnings.", "SKILL.md", ar_module, "AR2"), ], ) From 5f80cc2adfa5bb3849e2dde1755325f1d2c503e6 Mon Sep 17 00:00:00 2001 From: Rod Boev Date: Sat, 11 Jul 2026 00:28:36 -0400 Subject: [PATCH 5/5] fix(analyzer): keep executable doc calls outside suppression (#251) Signed-off-by: Rod Boev --- src/skillspector/nodes/analyzers/static_runner.py | 2 +- tests/nodes/analyzers/test_static_runner_filtering.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/skillspector/nodes/analyzers/static_runner.py b/src/skillspector/nodes/analyzers/static_runner.py index 8dd59719..f97ef8a4 100644 --- a/src/skillspector/nodes/analyzers/static_runner.py +++ b/src/skillspector/nodes/analyzers/static_runner.py @@ -182,7 +182,7 @@ def _is_eval_dataset(path: str) -> bool: _SEMANTIC_STRING_DOC_PRONE_RULES = frozenset({"PE3", "RA1", "TM1", "AR2"}) _EXECUTION_SIGNAL = re.compile( - r"(?:\b\w+\s*=|\bos\.(?:environ|getenv|system)\b|\b(?:subprocess|eval|exec)\b|[|>]" + r"(?:\b\w+\s*=|\bos\.(?:environ|getenv|system)\b|\bshutil\.rmtree\b|\b(?:subprocess|eval|exec)\b|[|>]" r"|\b(?:open|read_text|write_text)\s*\()", re.IGNORECASE, ) diff --git a/tests/nodes/analyzers/test_static_runner_filtering.py b/tests/nodes/analyzers/test_static_runner_filtering.py index f6bb798e..a69d7870 100644 --- a/tests/nodes/analyzers/test_static_runner_filtering.py +++ b/tests/nodes/analyzers/test_static_runner_filtering.py @@ -89,6 +89,7 @@ def test_comment_match_does_not_suppress_executable_twin(self) -> None: tm_module, "TM1", ), + ("shutil.rmtree('/')", "docs/cleanup.md", tm_module, "TM1"), ('/* note */ eval("modify this skill\'s configuration")', "note.js", ra_module, "RA1"), ("Do not include warnings.", "SKILL.md", ar_module, "AR2"), ],