diff --git a/.gitignore b/.gitignore index 41d1f9f6..dcf2e9ef 100644 --- a/.gitignore +++ b/.gitignore @@ -73,6 +73,9 @@ server/.generated/ tmp/ temp/ *.tmp +.codex-doc-qa/ +.codex-doc-work/ +.pnpm-store/ # OS .DS_Store @@ -87,4 +90,4 @@ Thumbs.db # Local notes rearch_plan.md -node_modules \ No newline at end of file +node_modules diff --git a/evaluators/builtin/pyproject.toml b/evaluators/builtin/pyproject.toml index f73240bb..be95d0fe 100644 --- a/evaluators/builtin/pyproject.toml +++ b/evaluators/builtin/pyproject.toml @@ -18,6 +18,7 @@ dependencies = [ galileo = ["agent-control-evaluator-galileo>=7.5.0"] budget = ["agent-control-evaluator-budget>=7.5.0"] cisco = ["agent-control-evaluator-cisco>=7.5.0"] +defenseclaw = ["agent-control-evaluator-defenseclaw>=8.2.0"] dev = ["pytest>=8.0.0", "pytest-asyncio>=0.23.0"] [project.entry-points."agent_control.evaluators"] @@ -39,3 +40,4 @@ agent-control-models = { workspace = true } agent-control-evaluator-galileo = { path = "../contrib/galileo", editable = true } agent-control-evaluator-budget = { path = "../contrib/budget", editable = true } agent-control-evaluator-cisco = { path = "../contrib/cisco", editable = true } +agent-control-evaluator-defenseclaw = { path = "../contrib/defenseclaw", editable = true } diff --git a/evaluators/builtin/tests/test_contrib_packages.py b/evaluators/builtin/tests/test_contrib_packages.py index 9f25186d..3ae1670f 100644 --- a/evaluators/builtin/tests/test_contrib_packages.py +++ b/evaluators/builtin/tests/test_contrib_packages.py @@ -39,6 +39,7 @@ def test_discover_contrib_packages_returns_expected_metadata() -> None: assert [(package.name, package.package, package.extra) for package in packages] == [ ("budget", "agent-control-evaluator-budget", "budget"), ("cisco", "agent-control-evaluator-cisco", "cisco"), + ("defenseclaw", "agent-control-evaluator-defenseclaw", "defenseclaw"), ("galileo", "agent-control-evaluator-galileo", "galileo"), ] diff --git a/evaluators/contrib/defenseclaw/Makefile b/evaluators/contrib/defenseclaw/Makefile new file mode 100644 index 00000000..ea744392 --- /dev/null +++ b/evaluators/contrib/defenseclaw/Makefile @@ -0,0 +1,33 @@ +.PHONY: help sync test lint lint-fix typecheck check build + +PACKAGE := agent-control-evaluator-defenseclaw + +help: + @echo "Agent Control Evaluator - DefenseClaw - Makefile commands" + @echo " make sync - sync package dependencies" + @echo " make test - run pytest" + @echo " make lint - run ruff check" + @echo " make lint-fix - run ruff check --fix" + @echo " make typecheck - run mypy" + @echo " make check - run tests, lint, and typecheck" + @echo " make build - build package" + +sync: + uv sync + +test: + uv run --with pytest --with pytest-asyncio --with pytest-cov --package $(PACKAGE) pytest tests --cov=src --cov-report=xml:../../../coverage-evaluators-defenseclaw.xml -q + +lint: + uv run --with ruff --package $(PACKAGE) ruff check --config ../../../pyproject.toml src/ tests/ + +lint-fix: + uv run --with ruff --package $(PACKAGE) ruff check --config ../../../pyproject.toml --fix src/ tests/ + +typecheck: + uv run --with mypy --package $(PACKAGE) mypy --config-file ../../../pyproject.toml src/ + +check: test lint typecheck + +build: + uv build diff --git a/evaluators/contrib/defenseclaw/README.md b/evaluators/contrib/defenseclaw/README.md new file mode 100644 index 00000000..cfb6dffb --- /dev/null +++ b/evaluators/contrib/defenseclaw/README.md @@ -0,0 +1,16 @@ +# DefenseClaw evaluators + +This package exposes two global external evaluators for Agent Control: + +- `defenseclaw.rule_pack` validates a versioned DefenseClaw rule pack. +- `defenseclaw.opa_policy` validates a versioned DefenseClaw OPA policy configuration. + +Install both evaluators with: + +```bash +pip install "agent-control-evaluators[defenseclaw]" +``` + +The configuration contracts and JSON Schemas are implemented and discoverable. Both evaluator +classes intentionally execute as no-ops: they return `matched=False` without contacting or +installing any DefenseClaw runtime or OSS package. diff --git a/evaluators/contrib/defenseclaw/pyproject.toml b/evaluators/contrib/defenseclaw/pyproject.toml new file mode 100644 index 00000000..9c4264ee --- /dev/null +++ b/evaluators/contrib/defenseclaw/pyproject.toml @@ -0,0 +1,42 @@ +[project] +name = "agent-control-evaluator-defenseclaw" +version = "8.2.0" +description = "DefenseClaw evaluators for agent-control" +readme = "README.md" +requires-python = ">=3.12" +license = { text = "Apache-2.0" } +authors = [{ name = "Agent Control Team" }] +dependencies = [ + "agent-control-evaluators>=8.2.0", + "agent-control-models>=8.2.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0.0", + "pytest-asyncio>=0.23.0", + "pytest-cov>=4.0.0", + "ruff>=0.1.0", + "mypy>=1.8.0", +] + +[project.entry-points."agent_control.evaluators"] +"defenseclaw.rule_pack" = "agent_control_evaluator_defenseclaw.rule_pack:DefenseClawRulePackEvaluator" +"defenseclaw.opa_policy" = "agent_control_evaluator_defenseclaw.opa_policy:DefenseClawOpaPolicyEvaluator" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/agent_control_evaluator_defenseclaw"] + +[tool.uv.sources] +agent-control-evaluators = { path = "../../builtin", editable = true } +agent-control-models = { path = "../../../models", editable = true } + +[dependency-groups] +dev = [ + "pytest>=9.0.2", + "pytest-asyncio>=1.3.0", +] diff --git a/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/__init__.py b/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/__init__.py new file mode 100644 index 00000000..ee023ded --- /dev/null +++ b/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/__init__.py @@ -0,0 +1,28 @@ +"""Agent Control DefenseClaw external evaluators.""" + +from importlib.metadata import PackageNotFoundError, version + +try: + __version__ = version("agent-control-evaluator-defenseclaw") +except PackageNotFoundError: + __version__ = "0.0.0.dev" + +from .common import Severity +from .opa_policy import DefenseClawOpaPolicyConfig, DefenseClawOpaPolicyEvaluator, OpaPolicy +from .rule_pack import ( + DefenseClawRulePackConfig, + DefenseClawRulePackEvaluator, + RuleConfig, + RulePack, +) + +__all__ = [ + "DefenseClawOpaPolicyConfig", + "DefenseClawOpaPolicyEvaluator", + "DefenseClawRulePackConfig", + "DefenseClawRulePackEvaluator", + "OpaPolicy", + "RuleConfig", + "RulePack", + "Severity", +] diff --git a/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/common.py b/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/common.py new file mode 100644 index 00000000..c1250b31 --- /dev/null +++ b/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/common.py @@ -0,0 +1,22 @@ +"""Shared DefenseClaw evaluator types and runtime behavior.""" + +from __future__ import annotations + +from typing import Annotated, Literal + +from agent_control_models import EvaluatorResult +from pydantic import ConfigDict, StringConstraints + +Severity = Literal["LOW", "MEDIUM", "HIGH", "CRITICAL"] +NonEmptyString = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] + +STRICT_PROVIDER_CONFIG = ConfigDict(extra="forbid") + + +def no_op_result() -> EvaluatorResult: + """Return the intentional no-op evaluation result.""" + return EvaluatorResult( + matched=False, + confidence=1.0, + message="DefenseClaw configuration accepted; evaluator execution is a no-op", + ) diff --git a/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/opa_policy/__init__.py b/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/opa_policy/__init__.py new file mode 100644 index 00000000..b5710c0c --- /dev/null +++ b/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/opa_policy/__init__.py @@ -0,0 +1,6 @@ +"""DefenseClaw OPA-policy evaluator exports.""" + +from .config import DefenseClawOpaPolicyConfig, OpaPolicy +from .evaluator import DefenseClawOpaPolicyEvaluator + +__all__ = ["DefenseClawOpaPolicyConfig", "DefenseClawOpaPolicyEvaluator", "OpaPolicy"] diff --git a/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/opa_policy/config.py b/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/opa_policy/config.py new file mode 100644 index 00000000..ea125d30 --- /dev/null +++ b/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/opa_policy/config.py @@ -0,0 +1,30 @@ +"""Typed configuration for the DefenseClaw OPA-policy evaluator.""" + +from __future__ import annotations + +from typing import Literal + +from agent_control_evaluators import EvaluatorConfig +from pydantic import BaseModel + +from ..common import STRICT_PROVIDER_CONFIG, Severity + + +class OpaPolicy(BaseModel): + """DefenseClaw OPA policy settings supported by the v1 contract.""" + + model_config = STRICT_PROVIDER_CONFIG + + domain: Literal["guardrail"] = "guardrail" + block_at: Severity = "HIGH" + alert_at: Severity = "MEDIUM" + cisco_trust_level: Literal["full"] = "full" + + +class DefenseClawOpaPolicyConfig(EvaluatorConfig): + """Configuration envelope for `defenseclaw.opa_policy`.""" + + model_config = STRICT_PROVIDER_CONFIG + + schema_version: Literal[1] = 1 + policy: OpaPolicy diff --git a/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/opa_policy/evaluator.py b/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/opa_policy/evaluator.py new file mode 100644 index 00000000..a0649b1c --- /dev/null +++ b/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/opa_policy/evaluator.py @@ -0,0 +1,35 @@ +"""DefenseClaw OPA-policy evaluator.""" + +from __future__ import annotations + +from importlib.metadata import PackageNotFoundError, version +from typing import Any + +from agent_control_evaluators import Evaluator, EvaluatorMetadata, register_evaluator +from agent_control_models import EvaluatorResult + +from ..common import no_op_result +from .config import DefenseClawOpaPolicyConfig + + +def _resolve_package_version() -> str: + try: + return version("agent-control-evaluator-defenseclaw") + except PackageNotFoundError: + return "0.0.0.dev" + + +@register_evaluator +class DefenseClawOpaPolicyEvaluator(Evaluator[DefenseClawOpaPolicyConfig]): + """Evaluate selected data with a typed DefenseClaw OPA policy.""" + + metadata = EvaluatorMetadata( + name="defenseclaw.opa_policy", + version=_resolve_package_version(), + description="DefenseClaw OPA-policy evaluation", + ) + config_model = DefenseClawOpaPolicyConfig + + async def evaluate(self, data: Any) -> EvaluatorResult: + """Return the intentional no-op result for all selected data.""" + return no_op_result() diff --git a/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/py.typed b/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/rule_pack/__init__.py b/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/rule_pack/__init__.py new file mode 100644 index 00000000..a034876e --- /dev/null +++ b/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/rule_pack/__init__.py @@ -0,0 +1,11 @@ +"""DefenseClaw rule-pack evaluator exports.""" + +from .config import DefenseClawRulePackConfig, RuleConfig, RulePack +from .evaluator import DefenseClawRulePackEvaluator + +__all__ = [ + "DefenseClawRulePackConfig", + "DefenseClawRulePackEvaluator", + "RuleConfig", + "RulePack", +] diff --git a/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/rule_pack/config.py b/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/rule_pack/config.py new file mode 100644 index 00000000..c0699559 --- /dev/null +++ b/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/rule_pack/config.py @@ -0,0 +1,50 @@ +"""Typed configuration for the DefenseClaw rule-pack evaluator.""" + +from __future__ import annotations + +from typing import Literal, Self + +from agent_control_evaluators import EvaluatorConfig +from pydantic import BaseModel, Field, model_validator + +from ..common import STRICT_PROVIDER_CONFIG, NonEmptyString, Severity + + +class RuleConfig(BaseModel): + """One DefenseClaw rule.""" + + model_config = STRICT_PROVIDER_CONFIG + + id: NonEmptyString + pattern: NonEmptyString + title: NonEmptyString + severity: Severity + confidence: float = Field(ge=0.0, le=1.0) + tags: list[NonEmptyString] + + +class RulePack(BaseModel): + """Versioned DefenseClaw rule pack.""" + + model_config = STRICT_PROVIDER_CONFIG + + version: Literal[1] = 1 + category: Literal["agent-control"] = "agent-control" + rules: list[RuleConfig] = Field(min_length=1) + + @model_validator(mode="after") + def validate_unique_rule_ids(self) -> Self: + """Reject ambiguous rule packs containing duplicate identifiers.""" + rule_ids = [rule.id for rule in self.rules] + if len(rule_ids) != len(set(rule_ids)): + raise ValueError("rule ids must be unique within a rule pack") + return self + + +class DefenseClawRulePackConfig(EvaluatorConfig): + """Configuration envelope for `defenseclaw.rule_pack`.""" + + model_config = STRICT_PROVIDER_CONFIG + + schema_version: Literal[1] = 1 + rule_pack: RulePack diff --git a/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/rule_pack/evaluator.py b/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/rule_pack/evaluator.py new file mode 100644 index 00000000..d48decd4 --- /dev/null +++ b/evaluators/contrib/defenseclaw/src/agent_control_evaluator_defenseclaw/rule_pack/evaluator.py @@ -0,0 +1,35 @@ +"""DefenseClaw rule-pack evaluator.""" + +from __future__ import annotations + +from importlib.metadata import PackageNotFoundError, version +from typing import Any + +from agent_control_evaluators import Evaluator, EvaluatorMetadata, register_evaluator +from agent_control_models import EvaluatorResult + +from ..common import no_op_result +from .config import DefenseClawRulePackConfig + + +def _resolve_package_version() -> str: + try: + return version("agent-control-evaluator-defenseclaw") + except PackageNotFoundError: + return "0.0.0.dev" + + +@register_evaluator +class DefenseClawRulePackEvaluator(Evaluator[DefenseClawRulePackConfig]): + """Evaluate selected data with a typed DefenseClaw rule pack.""" + + metadata = EvaluatorMetadata( + name="defenseclaw.rule_pack", + version=_resolve_package_version(), + description="DefenseClaw rule-pack evaluation", + ) + config_model = DefenseClawRulePackConfig + + async def evaluate(self, data: Any) -> EvaluatorResult: + """Return the intentional no-op result for all selected data.""" + return no_op_result() diff --git a/evaluators/contrib/defenseclaw/tests/__init__.py b/evaluators/contrib/defenseclaw/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/evaluators/contrib/defenseclaw/tests/test_configs.py b/evaluators/contrib/defenseclaw/tests/test_configs.py new file mode 100644 index 00000000..28994115 --- /dev/null +++ b/evaluators/contrib/defenseclaw/tests/test_configs.py @@ -0,0 +1,155 @@ +"""Contract tests for the DefenseClaw evaluator configurations.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from agent_control_evaluator_defenseclaw import ( + DefenseClawOpaPolicyConfig, + DefenseClawRulePackConfig, +) + + +def _rule_pack_config() -> dict[str, object]: + return { + "schema_version": 1, + "rule_pack": { + "version": 1, + "category": "agent-control", + "rules": [ + { + "id": "AC-CMD-RM-RF", + "pattern": "rm\\s+-rf", + "title": "Recursive deletion", + "severity": "HIGH", + "confidence": 0.99, + "tags": ["filesystem"], + } + ], + }, + } + + +def test_rule_pack_example_round_trips_with_fixed_fields() -> None: + # Given: the supplied rule-pack example + config = _rule_pack_config() + + # When: validating and serializing it + serialized = DefenseClawRulePackConfig.model_validate(config).model_dump() + + # Then: fixed fields and nested rules are preserved + assert serialized == config + + +def test_rule_pack_defaults_are_serialized() -> None: + # Given: a rule pack omitting fields with fixed v1 defaults + config = _rule_pack_config() + config.pop("schema_version") + rule_pack = config["rule_pack"] + assert isinstance(rule_pack, dict) + rule_pack.pop("version") + rule_pack.pop("category") + + # When: validating and serializing it + serialized = DefenseClawRulePackConfig.model_validate(config).model_dump() + + # Then: the wire-contract constants are materialized + assert serialized["schema_version"] == 1 + assert serialized["rule_pack"]["version"] == 1 + assert serialized["rule_pack"]["category"] == "agent-control" + + +@pytest.mark.parametrize( + ("path", "value"), + [ + (("schema_version",), 2), + (("rule_pack", "version"), 2), + (("rule_pack", "category"), "other"), + (("rule_pack", "rules", 0, "severity"), "URGENT"), + (("rule_pack", "rules", 0, "confidence"), -0.1), + (("rule_pack", "rules", 0, "confidence"), 1.1), + ], +) +def test_rule_pack_rejects_unsupported_values(path: tuple[object, ...], value: object) -> None: + # Given: a valid config with one unsupported nested value + config = _rule_pack_config() + target: object = config + for part in path[:-1]: + target = target[part] # type: ignore[index] + target[path[-1]] = value # type: ignore[index] + + # When/Then: typed validation rejects it + with pytest.raises(ValidationError): + DefenseClawRulePackConfig.model_validate(config) + + +def test_rule_pack_rejects_empty_and_duplicate_rules() -> None: + # Given: an empty rule list + empty = _rule_pack_config() + empty["rule_pack"]["rules"] = [] # type: ignore[index] + + # When/Then: at least one rule is required + with pytest.raises(ValidationError): + DefenseClawRulePackConfig.model_validate(empty) + + # Given: duplicate rule IDs + duplicate = _rule_pack_config() + first_rule = duplicate["rule_pack"]["rules"][0] # type: ignore[index] + duplicate["rule_pack"]["rules"].append(dict(first_rule)) # type: ignore[index,union-attr] + + # When/Then: duplicate IDs are rejected + with pytest.raises(ValidationError, match="rule ids must be unique"): + DefenseClawRulePackConfig.model_validate(duplicate) + + +def test_rule_pack_rejects_blank_strings_and_unknown_fields() -> None: + # Given: whitespace-only required values and an unknown provider key + config = _rule_pack_config() + rule = config["rule_pack"]["rules"][0] # type: ignore[index] + rule["id"] = " " + rule["unknown"] = True + + # When/Then: strict validation rejects both errors + with pytest.raises(ValidationError) as exc_info: + DefenseClawRulePackConfig.model_validate(config) + errors = exc_info.value.errors() + assert any(error["loc"][-1] == "id" for error in errors) + assert any(error["loc"][-1] == "unknown" for error in errors) + + +def test_opa_policy_example_and_defaults() -> None: + # Given: the supplied policy example + config = { + "schema_version": 1, + "policy": { + "domain": "guardrail", + "block_at": "HIGH", + "alert_at": "MEDIUM", + "cisco_trust_level": "full", + }, + } + + # When: validating it and validating a default policy + serialized = DefenseClawOpaPolicyConfig.model_validate(config).model_dump() + defaults = DefenseClawOpaPolicyConfig.model_validate({"policy": {}}).model_dump() + + # Then: both serialize to the canonical policy envelope + assert serialized == config + assert defaults == config + + +@pytest.mark.parametrize( + "policy", + [ + {"domain": "other"}, + {"block_at": "URGENT"}, + {"alert_at": "URGENT"}, + {"cisco_trust_level": "partial"}, + {"unexpected": True}, + ], +) +def test_opa_policy_rejects_unsupported_values(policy: dict[str, object]) -> None: + # Given/When/Then: an unsupported provider value is rejected + with pytest.raises(ValidationError): + DefenseClawOpaPolicyConfig.model_validate({"policy": policy}) diff --git a/evaluators/contrib/defenseclaw/tests/test_evaluators.py b/evaluators/contrib/defenseclaw/tests/test_evaluators.py new file mode 100644 index 00000000..8bfaaf55 --- /dev/null +++ b/evaluators/contrib/defenseclaw/tests/test_evaluators.py @@ -0,0 +1,85 @@ +"""Behavior tests for the DefenseClaw no-op evaluators.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path + +import pytest + +from agent_control_evaluator_defenseclaw import ( + DefenseClawOpaPolicyEvaluator, + DefenseClawRulePackEvaluator, +) + + +def _rule_pack_config() -> dict[str, object]: + return { + "rule_pack": { + "rules": [ + { + "id": "rule-1", + "pattern": "example", + "title": "Example", + "severity": "LOW", + "confidence": 1.0, + "tags": [], + } + ] + } + } + + +def test_metadata_names_are_distinct_public_identifiers() -> None: + assert DefenseClawRulePackEvaluator.metadata.name == "defenseclaw.rule_pack" + assert DefenseClawOpaPolicyEvaluator.metadata.name == "defenseclaw.opa_policy" + assert DefenseClawRulePackEvaluator is not DefenseClawOpaPolicyEvaluator + + +def test_entry_points_match_public_metadata_names() -> None: + # Given: the installed-package manifest contract + manifest_path = Path(__file__).parent.parent / "pyproject.toml" + manifest = tomllib.loads(manifest_path.read_text()) + + # When: reading evaluator entry points + entry_points = manifest["project"]["entry-points"]["agent_control.evaluators"] + + # Then: both exact global names resolve to their distinct classes + assert entry_points == { + "defenseclaw.rule_pack": ( + "agent_control_evaluator_defenseclaw.rule_pack:DefenseClawRulePackEvaluator" + ), + "defenseclaw.opa_policy": ( + "agent_control_evaluator_defenseclaw.opa_policy:DefenseClawOpaPolicyEvaluator" + ), + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize("data", [None, "", "selected data", {"tool": "rm"}]) +async def test_rule_pack_evaluation_is_a_no_op(data: object) -> None: + # Given: a valid rule-pack evaluator + evaluator = DefenseClawRulePackEvaluator.from_dict(_rule_pack_config()) + + # When: evaluating any selected data + result = await evaluator.evaluate(data) + + # Then: execution is intentionally inert and healthy + assert result.matched is False + assert result.confidence == 1.0 + assert result.error is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("data", [None, "", "selected data", {"tool": "rm"}]) +async def test_opa_policy_evaluation_is_a_no_op(data: object) -> None: + # Given: a valid OPA-policy evaluator + evaluator = DefenseClawOpaPolicyEvaluator.from_dict({"policy": {}}) + + # When: evaluating any selected data + result = await evaluator.evaluate(data) + + # Then: execution is intentionally inert and healthy + assert result.matched is False + assert result.confidence == 1.0 + assert result.error is None diff --git a/pyproject.toml b/pyproject.toml index b4af0eea..3cfb63c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,7 @@ version_toml = [ "evaluators/builtin/pyproject.toml:project.version", "evaluators/contrib/budget/pyproject.toml:project.version", "evaluators/contrib/cisco/pyproject.toml:project.version", + "evaluators/contrib/defenseclaw/pyproject.toml:project.version", "evaluators/contrib/galileo/pyproject.toml:project.version", ] version_source = "tag" diff --git a/ui/src/core/evaluators/defenseclaw/index.ts b/ui/src/core/evaluators/defenseclaw/index.ts new file mode 100644 index 00000000..c531bcaf --- /dev/null +++ b/ui/src/core/evaluators/defenseclaw/index.ts @@ -0,0 +1,140 @@ +import type { EvaluatorDefinition } from '../types'; +import { DefenseClawOpaPolicyForm } from './opa-policy-form'; +import { + createBlankDefenseClawRule, + DefenseClawRulePackForm, +} from './rule-pack-form'; +import { + DEFENSECLAW_SEVERITIES, + type DefenseClawOpaPolicyFormValues, + type DefenseClawRuleFormValue, + type DefenseClawRulePackFormValues, +} from './types'; + +const severityValues = new Set(DEFENSECLAW_SEVERITIES); + +const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); + +const normalizeRule = (value: unknown): DefenseClawRuleFormValue => { + if (!isRecord(value)) return createBlankDefenseClawRule(); + return { + id: typeof value.id === 'string' ? value.id : '', + pattern: typeof value.pattern === 'string' ? value.pattern : '', + title: typeof value.title === 'string' ? value.title : '', + severity: severityValues.has(String(value.severity)) + ? (value.severity as DefenseClawRuleFormValue['severity']) + : 'HIGH', + confidence: typeof value.confidence === 'number' ? value.confidence : 0.99, + tags: Array.isArray(value.tags) + ? value.tags.filter((tag): tag is string => typeof tag === 'string') + : [], + }; +}; + +const validateRules = (value: unknown): string | null => { + if (!Array.isArray(value) || value.length === 0) { + return 'At least one rule is required'; + } + const rules = value as DefenseClawRuleFormValue[]; + const ids = rules.map((rule) => rule.id.trim()); + if ( + rules.some( + (rule) => !rule.id.trim() || !rule.title.trim() || !rule.pattern.trim() + ) + ) { + return 'Every rule requires an ID, title, and pattern'; + } + if (new Set(ids).size !== ids.length) return 'Rule IDs must be unique'; + if ( + rules.some( + (rule) => + !Number.isFinite(rule.confidence) || + rule.confidence < 0 || + rule.confidence > 1 + ) + ) { + return 'Rule confidence must be between 0 and 1'; + } + if (rules.some((rule) => !severityValues.has(rule.severity))) { + return 'Every rule requires a supported severity'; + } + if (rules.some((rule) => rule.tags.some((tag) => !tag.trim()))) { + return 'Rule tags cannot be blank'; + } + return null; +}; + +export const defenseClawRulePackEvaluator: EvaluatorDefinition = + { + id: 'defenseclaw.rule_pack', + displayName: 'DefenseClaw Rule Pack', + initialValues: { rules: [createBlankDefenseClawRule()] }, + validate: { rules: validateRules }, + toConfig: (values) => ({ + schema_version: 1, + rule_pack: { + version: 1, + category: 'agent-control', + rules: values.rules, + }, + }), + fromConfig: (config) => { + const rulePack = isRecord(config.rule_pack) ? config.rule_pack : {}; + const rules = Array.isArray(rulePack.rules) + ? rulePack.rules.map(normalizeRule) + : []; + return { + rules: rules.length > 0 ? rules : [createBlankDefenseClawRule()], + }; + }, + FormComponent: DefenseClawRulePackForm, + }; + +export const defenseClawOpaPolicyEvaluator: EvaluatorDefinition = + { + id: 'defenseclaw.opa_policy', + displayName: 'DefenseClaw OPA Policy', + initialValues: { + domain: 'guardrail', + block_at: 'HIGH', + alert_at: 'MEDIUM', + cisco_trust_level: 'full', + }, + validate: { + domain: (value) => + value === 'guardrail' ? null : 'Domain must be guardrail', + block_at: (value) => + severityValues.has(String(value)) + ? null + : 'Select a supported block severity', + alert_at: (value) => + severityValues.has(String(value)) + ? null + : 'Select a supported alert severity', + cisco_trust_level: (value) => + value === 'full' ? null : 'Cisco trust level must be full', + }, + toConfig: (values) => ({ schema_version: 1, policy: { ...values } }), + fromConfig: (config) => { + const policy = isRecord(config.policy) ? config.policy : {}; + return { + domain: 'guardrail', + block_at: severityValues.has(String(policy.block_at)) + ? (policy.block_at as DefenseClawOpaPolicyFormValues['block_at']) + : 'HIGH', + alert_at: severityValues.has(String(policy.alert_at)) + ? (policy.alert_at as DefenseClawOpaPolicyFormValues['alert_at']) + : 'MEDIUM', + cisco_trust_level: 'full', + }; + }, + FormComponent: DefenseClawOpaPolicyForm, + }; + +export type { + DefenseClawOpaPolicyFormValues, + DefenseClawRuleFormValue, + DefenseClawRulePackFormValues, + DefenseClawSeverity, +} from './types'; diff --git a/ui/src/core/evaluators/defenseclaw/opa-policy-form.tsx b/ui/src/core/evaluators/defenseclaw/opa-policy-form.tsx new file mode 100644 index 00000000..f75167bc --- /dev/null +++ b/ui/src/core/evaluators/defenseclaw/opa-policy-form.tsx @@ -0,0 +1,48 @@ +import { Divider, Select, Stack } from '@mantine/core'; + +import type { EvaluatorFormProps } from '../types'; +import { + DEFENSECLAW_SEVERITIES, + type DefenseClawOpaPolicyFormValues, +} from './types'; + +const severityOptions = DEFENSECLAW_SEVERITIES.map((severity) => ({ + value: severity, + label: severity, +})); + +export const DefenseClawOpaPolicyForm = ({ + form, +}: EvaluatorFormProps) => ( + + + + + +); diff --git a/ui/src/core/evaluators/defenseclaw/rule-pack-form.tsx b/ui/src/core/evaluators/defenseclaw/rule-pack-form.tsx new file mode 100644 index 00000000..9d1aa2a4 --- /dev/null +++ b/ui/src/core/evaluators/defenseclaw/rule-pack-form.tsx @@ -0,0 +1,133 @@ +import { + ActionIcon, + Box, + Divider, + Group, + NumberInput, + Paper, + Select, + Stack, + TagsInput, + Text, + Textarea, + TextInput, +} from '@mantine/core'; +import { Button } from '@rungalileo/jupiter-ds'; +import { IconTrash } from '@tabler/icons-react'; + +import type { EvaluatorFormProps } from '../types'; +import { + DEFENSECLAW_SEVERITIES, + type DefenseClawRuleFormValue, + type DefenseClawRulePackFormValues, +} from './types'; + +export const createBlankDefenseClawRule = (): DefenseClawRuleFormValue => ({ + id: '', + pattern: '', + title: '', + severity: 'HIGH', + confidence: 0.99, + tags: [], +}); + +const severityOptions = DEFENSECLAW_SEVERITIES.map((severity) => ({ + value: severity, + label: severity, +})); + +export const DefenseClawRulePackForm = ({ + form, +}: EvaluatorFormProps) => ( + + + + {form.values.rules.map((rule, index) => ( + + + + + Rule {index + 1} + + form.removeListItem('rules', index)} + > + + + + + + +