diff --git a/rulechef/executor.py b/rulechef/executor.py index ecb8b94..0eb57f1 100644 --- a/rulechef/executor.py +++ b/rulechef/executor.py @@ -3,6 +3,7 @@ import json import re import signal +import string import threading from typing import Any @@ -18,6 +19,13 @@ # bound, one bad rule freezes the whole evaluation loop. RULE_TIMEOUT_S = 3.0 +# Characters trimmed from the edges of a produced entity span when snapping +# to word boundaries (see `substitute_template`). Whitespace + standard +# punctuation, not full Unicode punctuation categories — this only needs to +# catch the common over-capture cases (trailing comma, wrapping quotes), +# not act as a general tokenizer. +_BOUNDARY_TRIM_CHARS = string.whitespace + string.punctuation + class _RuleTimeout(Exception): pass @@ -183,6 +191,29 @@ def substitute_template( result["start"] = start + offset result["end"] = result["start"] + len(result["text"]) + # Snap the span to word boundaries: a regex that over-captures a + # trailing/leading punctuation or space mark (e.g. "London," instead of + # "London") turns a boundary slip into both a false positive and a false + # negative under the default `text` matching mode (see evaluation.py + # `_match_entities`). Trim only whitespace/punctuation at the edges — + # never touch interior characters — so the produced text lines up with + # gold spans without changing what the rule's capture group intended. + if ( + "text" in result + and "start" in result + and "end" in result + and isinstance(result["text"], str) + and isinstance(result["start"], int) + and isinstance(result["end"], int) + and result["text"] + ): + stripped = result["text"].strip(_BOUNDARY_TRIM_CHARS) + if stripped and stripped != result["text"]: + lead_trim = len(result["text"]) - len(result["text"].lstrip(_BOUNDARY_TRIM_CHARS)) + result["start"] += lead_trim + result["end"] = result["start"] + len(stripped) + result["text"] = stripped + return result diff --git a/tests/test_executor.py b/tests/test_executor.py index 26f28d7..23e33a5 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -49,6 +49,33 @@ def test_group_zero_via_dollar_zero(self): result = substitute_template(tpl, "full_match", 0, 10, groups=()) assert result == {"match": "full_match"} + def test_trailing_punctuation_snapped_from_span(self): + tpl = {"text": "$0", "start": "$start", "end": "$end"} + # "London," at offset 10-17 in some larger text; the rule over-captured + # the trailing comma. + result = substitute_template(tpl, "London,", 10, 17) + assert result == {"text": "London", "start": 10, "end": 16} + + def test_leading_and_trailing_whitespace_snapped_from_span(self): + tpl = {"text": "$0", "start": "$start", "end": "$end"} + result = substitute_template(tpl, " Paris ", 5, 12) + assert result == {"text": "Paris", "start": 6, "end": 11} + + def test_wrapping_quotes_snapped_from_span(self): + tpl = {"text": "$0", "start": "$start", "end": "$end"} + result = substitute_template(tpl, '"Berlin"', 0, 8) + assert result == {"text": "Berlin", "start": 1, "end": 7} + + def test_no_snap_needed_leaves_span_unchanged(self): + tpl = {"text": "$0", "start": "$start", "end": "$end"} + result = substitute_template(tpl, "Tokyo", 0, 5) + assert result == {"text": "Tokyo", "start": 0, "end": 5} + + def test_all_punctuation_text_is_not_snapped_to_empty(self): + tpl = {"text": "$0", "start": "$start", "end": "$end"} + result = substitute_template(tpl, "---", 0, 3) + assert result == {"text": "---", "start": 0, "end": 3} + # ========================================================================= # RuleExecutor._execute_regex_rule @@ -122,6 +149,26 @@ def test_no_match_returns_empty(self): results = self.executor._execute_regex_rule(rule, {"text": "nothing here"}) assert results == [] + def test_over_captured_trailing_punctuation_is_snapped(self): + # A rule that (deliberately, as a stand-in for a real over-broad + # LLM-written pattern) captures a trailing comma along with the city + # name. Under the default `text` matching mode this boundary slip + # would count as both a false positive and a false negative against + # a gold span of just "London" (rulechef/evaluation.py:_match_entities). + rule = Rule( + id="r5", + name="city", + description="Match a city mention followed by optional punctuation", + format=RuleFormat.REGEX, + content=r"London,?", + output_template={"text": "$0", "start": "$start", "end": "$end"}, + ) + text = "She lives in London, near the river." + results = self.executor._execute_regex_rule(rule, {"text": text}) + assert len(results) == 1 + assert results[0]["text"] == "London" + assert text[results[0]["start"] : results[0]["end"]] == "London" + # ========================================================================= # RuleExecutor._execute_code_rule