Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 54 additions & 9 deletions tests/test_loop.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,9 +182,11 @@ def create_message(self, payload, stream):
class FakeClientTwoEmptyThenDone:
def __init__(self):
self.calls = 0
self.payloads = []

def create_message(self, payload, stream):
self.calls += 1
self.payloads.append(payload)
if self.calls <= 2:
return {
"id": str(self.calls),
Expand All @@ -203,9 +205,11 @@ def create_message(self, payload, stream):
class FakeClientThreeEmpty:
def __init__(self):
self.calls = 0
self.payloads = []

def create_message(self, payload, stream):
self.calls += 1
self.payloads.append(payload)
return {
"id": str(self.calls),
"role": "assistant",
Expand All @@ -214,23 +218,54 @@ def create_message(self, payload, stream):
}


def test_loop_retries_on_empty_assistant_turn(tmp_path: Path):
def test_single_empty_turn_does_not_stall_run(tmp_path: Path):
client = FakeClientEmptyThenDone()
runner = Runner(client=client, repo=tmp_path, model="m", stream=False)

result = runner.run("whatever")

assert client.calls == 2
assert result["response"]["content"] == [{"type": "text", "text": "ok continuing"}]
assert client.second_payload is not None
continuation_messages = [
m
for m in client.second_payload["messages"]
if m["role"] == "user"
and m["content"]
and "Continue. You ended your previous turn with no output." in m["content"][0].get("text", "")
assert result["response"]["content"][0]["text"] == "ok continuing"


class FakeClientEmptyToolEmptyThenDone:
def __init__(self):
self.calls = 0
self.payloads = []

def create_message(self, payload, stream):
self.calls += 1
self.payloads.append(payload)
if self.calls == 1:
return {"id": "1", "role": "assistant", "content": []}
if self.calls == 2:
return {
"id": "2",
"role": "assistant",
"content": [{"type": "tool_use", "id": "tool-1", "name": "Ls", "input": {"path": "."}}],
}
if self.calls == 3:
return {"id": "3", "role": "assistant", "content": [{"type": "text", "text": " "}]}
return {"id": "4", "role": "assistant", "content": [{"type": "text", "text": "done"}]}


def test_empty_turn_counter_resets_after_tool_turn(tmp_path: Path):
client = FakeClientEmptyToolEmptyThenDone()
runner = Runner(client=client, repo=tmp_path, model="m", stream=False)

result = runner.run("list and continue")

assert client.calls == 4
assert result["response"]["content"] == [{"type": "text", "text": "done"}]
all_texts = [
b.get("text", "")
for payload in client.payloads
for message in payload.get("messages", [])
for b in message.get("content", [])
if isinstance(b, dict) and b.get("type") == "text"
]
assert continuation_messages
assert not any("RECOVERY: blank turns detected." in text for text in all_texts)


def test_tool_result_followup_is_pure_tool_result_message(tmp_path: Path):
Expand Down Expand Up @@ -336,6 +371,14 @@ def test_loop_retries_twice_then_succeeds(tmp_path: Path):

assert client.calls == 3
assert result["response"]["content"] == [{"type": "text", "text": "ok after two empties"}]
recovery_messages = [
b.get("text", "")
for m in client.payloads[2]["messages"]
if m.get("role") == "user"
for b in m.get("content", [])
if isinstance(b, dict) and b.get("type") == "text"
]
assert any("RECOVERY: blank turns detected." in text for text in recovery_messages)


def test_loop_stops_after_retry_limit_on_empty_turns(tmp_path: Path):
Expand All @@ -345,6 +388,8 @@ def test_loop_stops_after_retry_limit_on_empty_turns(tmp_path: Path):
result = runner.run("whatever")

assert client.calls == 3
assert result["execution"]["terminated_reason"] == "stalled_empty_responses"
assert result["execution"]["completed"] is False
assert result["response"]["content"] == []


Expand Down
86 changes: 86 additions & 0 deletions tests/test_state_runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,92 @@ def test_verification_detail_event_keeps_raw_trace(tmp_path: Path, monkeypatch:
assert "last_validation_summary:" in out


def test_git_changed_files_keeps_git_status_behavior(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
class Proc:
returncode = 0
stdout = " M src/a.py\n?? tests/test_a.py\n"

monkeypatch.setattr(state_runtime.subprocess, "run", lambda *args, **kwargs: Proc())
assert state_runtime.git_changed_files(tmp_path) == ["src/a.py", "tests/test_a.py"]


def test_git_changed_files_non_git_first_call_returns_empty(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
class Proc:
returncode = 128
stdout = ""

monkeypatch.setattr(state_runtime.subprocess, "run", lambda *args, **kwargs: Proc())
monkeypatch.setattr(state_runtime, "_NON_GIT_SNAPSHOT_BY_REPO", {})
assert state_runtime.git_changed_files(tmp_path) == []


def test_git_changed_files_non_git_detects_new_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
class Proc:
returncode = 128
stdout = ""

monkeypatch.setattr(state_runtime.subprocess, "run", lambda *args, **kwargs: Proc())
monkeypatch.setattr(state_runtime, "_NON_GIT_SNAPSHOT_BY_REPO", {})
assert state_runtime.git_changed_files(tmp_path) == []
(tmp_path / "created.txt").write_text("hello", encoding="utf-8")
assert state_runtime.git_changed_files(tmp_path) == ["created.txt"]


def test_git_changed_files_non_git_detects_modified_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
class Proc:
returncode = 128
stdout = ""

file_path = tmp_path / "file.txt"
file_path.write_text("one", encoding="utf-8")
monkeypatch.setattr(state_runtime.subprocess, "run", lambda *args, **kwargs: Proc())
monkeypatch.setattr(state_runtime, "_NON_GIT_SNAPSHOT_BY_REPO", {})
assert state_runtime.git_changed_files(tmp_path) == []
file_path.write_text("three", encoding="utf-8")
assert state_runtime.git_changed_files(tmp_path) == ["file.txt"]


def test_git_changed_files_non_git_detects_deleted_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
class Proc:
returncode = 128
stdout = ""

file_path = tmp_path / "deleted.txt"
file_path.write_text("x", encoding="utf-8")
monkeypatch.setattr(state_runtime.subprocess, "run", lambda *args, **kwargs: Proc())
monkeypatch.setattr(state_runtime, "_NON_GIT_SNAPSHOT_BY_REPO", {})
assert state_runtime.git_changed_files(tmp_path) == []
file_path.unlink()
assert state_runtime.git_changed_files(tmp_path) == ["deleted.txt"]


def test_git_changed_files_non_git_ignores_repo_ignored_paths(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
class Proc:
returncode = 128
stdout = ""

monkeypatch.setattr(state_runtime.subprocess, "run", lambda *args, **kwargs: Proc())
monkeypatch.setattr(state_runtime, "_NON_GIT_SNAPSHOT_BY_REPO", {})
assert state_runtime.git_changed_files(tmp_path) == []
(tmp_path / ".git").mkdir()
(tmp_path / ".git" / "config").write_text("x", encoding="utf-8")
assert state_runtime.git_changed_files(tmp_path) == []


def test_git_changed_files_non_git_returns_posix_relative_paths(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
class Proc:
returncode = 128
stdout = ""

monkeypatch.setattr(state_runtime.subprocess, "run", lambda *args, **kwargs: Proc())
monkeypatch.setattr(state_runtime, "_NON_GIT_SNAPSHOT_BY_REPO", {})
assert state_runtime.git_changed_files(tmp_path) == []
nested = tmp_path / "a" / "b"
nested.mkdir(parents=True)
(nested / "c.txt").write_text("x", encoding="utf-8")
assert state_runtime.git_changed_files(tmp_path) == ["a/b/c.txt"]


def test_repeated_validation_updates_compact_state_without_repeated_prose(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
Expand Down
59 changes: 59 additions & 0 deletions tests/test_state_tooling_mutation_policy.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from __future__ import annotations

import json
from pathlib import Path

from villani_code.state import Runner
Expand Down Expand Up @@ -126,3 +127,61 @@ def test_patch_existing_file_rewrite_heavy_is_rejected(tmp_path: Path) -> None:
result = execute_tool_with_policy(runner, "Patch", {"unified_diff": "\n".join(diff_lines)}, "1", 0)
assert result["is_error"] is True
assert "Rewrite-heavy mutation rejected" in str(result["content"])


def test_failed_bash_command_is_blocked_on_immediate_retry(tmp_path: Path) -> None:
runner = _runner(tmp_path)
command = "python -c \"import sys; sys.exit(1)\""
first = execute_tool_with_policy(runner, "Bash", {"command": command}, "1", 0)
assert first["is_error"] is False
assert json.loads(str(first["content"]))["exit_code"] != 0
second = execute_tool_with_policy(runner, "Bash", {"command": command}, "2", 0)
assert second["is_error"] is True
assert "already failed" in str(second["content"])


def test_failed_bash_retry_block_normalizes_whitespace(tmp_path: Path) -> None:
runner = _runner(tmp_path)
first = execute_tool_with_policy(runner, "Bash", {"command": " python -c \"import sys; sys.exit(1)\" "}, "1", 0)
assert json.loads(str(first["content"]))["exit_code"] != 0
second = execute_tool_with_policy(runner, "Bash", {"command": "python -c \"import sys; sys.exit(1)\""}, "2", 0)
assert second["is_error"] is True
assert "already failed" in str(second["content"])


def test_failed_bash_retry_block_is_case_insensitive_on_windows(tmp_path: Path) -> None:
runner = _runner(tmp_path)
runner._shell_retry_case_insensitive = True
first = execute_tool_with_policy(runner, "Bash", {"command": "PyThOn -c \"import sys; sys.exit(1)\""}, "1", 0)
assert json.loads(str(first["content"]))["exit_code"] != 0
second = execute_tool_with_policy(runner, "Bash", {"command": "python -C \"import sys; sys.exit(1)\""}, "2", 0)
assert second["is_error"] is True


def test_materially_different_bash_command_is_allowed_after_failure(tmp_path: Path) -> None:
runner = _runner(tmp_path)
failed = execute_tool_with_policy(runner, "Bash", {"command": "python -c \"import sys; sys.exit(1)\""}, "1", 0)
assert json.loads(str(failed["content"]))["exit_code"] != 0
allowed = execute_tool_with_policy(runner, "Bash", {"command": "python -c \"print('ok')\""}, "2", 0)
assert allowed["is_error"] is False
assert json.loads(str(allowed["content"]))["exit_code"] == 0


def test_successful_bash_command_clears_retry_block(tmp_path: Path) -> None:
runner = _runner(tmp_path)
command = "python -c \"import sys; sys.exit(1)\""
failed = execute_tool_with_policy(runner, "Bash", {"command": command}, "1", 0)
assert json.loads(str(failed["content"]))["exit_code"] != 0
success = execute_tool_with_policy(runner, "Bash", {"command": "python -c \"print('reset')\""}, "2", 0)
assert json.loads(str(success["content"]))["exit_code"] == 0
retry = execute_tool_with_policy(runner, "Bash", {"command": command}, "3", 0)
assert retry["is_error"] is False
assert json.loads(str(retry["content"]))["exit_code"] != 0


def test_non_shell_tools_are_unaffected_by_failed_shell_retry_memory(tmp_path: Path) -> None:
runner = _runner(tmp_path)
_ = execute_tool_with_policy(runner, "Bash", {"command": "python -c \"import sys; sys.exit(1)\""}, "1", 0)
read = execute_tool_with_policy(runner, "Read", {"file_path": "missing.txt"}, "2", 0)
assert read["is_error"] is True
assert "No such file" in str(read["content"])
Loading
Loading