diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cd89b6..500e0fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ under Unreleased and move into a version section at release time. ### Added +- `wildedge doctor --send-test-event`: sends one real span event through the full pipeline and reports the ingest response, proving DSN auth and connectivity end to end. The report gains an `environment` section (`WILDEDGE_*` variables, autoload PYTHONPATH status) plus `config_status` / `connectivity_status` fields. - `wildedge.llm_api()`: provider-agnostic tracking for LLM calls made with any HTTP client (OpenRouter, vLLM, Ollama, OpenAI/Anthropic-compatible endpoints). Times the block, normalizes usage payloads from either provider shape via `call.usage()` / `call.response()`, supports TTFT marks and async use, and records exceptions as error events. See `docs/llm_api.md`. - `register_model()` without a matching extractor defaults the model name to the explicit `model_id` instead of the placeholder object's type name. - Process-wide default client: `wildedge.get_client()`, `wildedge.set_default_client()`, and module-level `trace`, `span`, `track_span`, `register_model`, `flush` delegating to it (#41) @@ -18,6 +19,9 @@ under Unreleased and move into a version section at release time. ### Changed +- `wildedge doctor` exit codes are now differentiated: 0 pass, 1 configuration or dependency failure, 2 connectivity failure (`--network-check` or `--send-test-event`). A failing network check previously exited 1. +- The test suite isolates all default SDK state paths under tmp; tests no longer write to the machine-global state directory. + - `--strict-integrations` now takes effect: a failed required integration exits the process with code 121. Previously the enforcing code path was never invoked, so the flag was silently ignored. - `--print-startup-report` and `--no-propagate` now work under `wildedge run`; both were only wired to the unused runner path before. - Constructing a client without a DSN logs once per process at INFO. Previously every construction logged a WARNING. diff --git a/README.md b/README.md index ce4a41e..31379b6 100644 --- a/README.md +++ b/README.md @@ -30,10 +30,12 @@ WILDEDGE_DSN="https://@ingest.wildedge.dev/" \ wildedge run --integrations timm -- python app.py ``` -Validate your environment before deploying: +Validate your environment before deploying. `--send-test-event` proves the +whole pipeline end to end by sending one span event and reporting the ingest +response (exit codes: 0 pass, 1 config failure, 2 connectivity failure): ```bash -wildedge doctor --integrations all --network-check +wildedge doctor --integrations all --send-test-event ``` Useful flags: diff --git a/docs/deployment.md b/docs/deployment.md index 16e49aa..713c016 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -92,6 +92,27 @@ them unwrapped or unset `WILDEDGE_DSN` there. `wildedge.span(...)` blocks correlate into the same trace no matter which client emits them. +## Verifying a deployment + +`wildedge doctor` answers "will events actually flow from this machine" +before you rely on it: + +```bash +wildedge doctor --integrations all --send-test-event --format json +``` + +- `--send-test-event` sends one real span event through the full pipeline + (DSN auth, ingest endpoint, batch protocol) and reports the server's + response, which a TCP-level `--network-check` cannot do. +- The report includes an `environment` section with every `WILDEDGE_*` + variable the runtime reads, plus whether the autoload dir is on + `PYTHONPATH`. +- Exit codes: 0 all pass, 1 configuration or dependency failure, 2 the + config is fine but the ingest endpoint is unreachable or rejecting. + +`--format json` makes the output machine-readable; pasting it into an issue +or an AI assistant is the intended debugging flow. + ## Environment propagation By default, `wildedge run` leaves its `WILDEDGE_*` variables in the diff --git a/tests/conftest.py b/tests/conftest.py index d57f383..3c8890a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,6 +28,21 @@ def reset_default_client(): set_default_client(None) +@pytest.fixture(autouse=True) +def isolate_sdk_state(tmp_path, monkeypatch): + """Keep every default persistence path under tmp. + + Without this, tests that construct real clients write queues, registries + and dead letters to the machine-global SDK state dir, leaking state + between runs and machines (and once masking a real regression locally + while CI failed). + """ + import wildedge.paths as paths + + monkeypatch.setattr(paths, "default_sdk_state_dir", lambda: tmp_path / "state") + monkeypatch.setattr(paths, "default_sdk_cache_dir", lambda: tmp_path / "cache") + + PLATFORM_MARKS = { "requires_linux": "linux", "requires_macos": "darwin", diff --git a/tests/test_cli.py b/tests/test_cli.py index 0dd405c..8bb3cf3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -264,6 +264,9 @@ def test_doctor_json_output_schema(monkeypatch, capsys): assert rc == 0 assert sorted(payload.keys()) == [ "checks", + "config_status", + "connectivity_status", + "environment", "hubs", "integrations", "platform", @@ -284,7 +287,7 @@ def test_doctor_network_check_failure(monkeypatch, capsys): rc = cli.main(["doctor", "--network-check", "--integrations", "onnx"]) out = capsys.readouterr().out - assert rc == 1 + assert rc == 2 assert "network: FAIL (unreachable)" in out assert "doctor: FAIL" in out @@ -455,3 +458,130 @@ def close(self): # type: ignore[no-untyped-def] assert peek_default_client() is context.client finally: context.shutdown() + + +def _doctor_env(monkeypatch): + monkeypatch.setenv(constants.ENV_DSN, "https://secret@ingest.wildedge.dev/key") + monkeypatch.setattr(cli.importlib.util, "find_spec", lambda _: object()) + monkeypatch.setattr(cli, "check_writable_dir", lambda path: (True, str(path))) + + +def _fake_device(api_key, app_version, overrides=None): + from wildedge.platforms.device_info import DeviceInfo + + return DeviceInfo(device_id="d", device_type="linux") + + +def test_doctor_send_test_event_accepted(monkeypatch, capsys): + from wildedge.transmitter import IngestResponse + + _doctor_env(monkeypatch) + monkeypatch.setattr(cli, "detect_device", _fake_device) + sent_batches: list[dict] = [] + + class FakeTransmitter: + def __init__(self, api_key, host): + assert api_key == "secret" + + def send(self, batch): + sent_batches.append(batch) + return IngestResponse( + status="accepted", batch_id="b1", events_accepted=1, events_rejected=0 + ) + + monkeypatch.setattr(cli, "Transmitter", FakeTransmitter) + + rc = cli.main(["doctor", "--integrations", "onnx", "--send-test-event"]) + out = capsys.readouterr().out + + assert rc == 0 + assert "ingest: OK" in out + assert "connectivity: PASS" in out + batch = sent_batches[0] + assert batch["events"][0]["event_type"] == "span" + assert batch["events"][0]["span"]["name"] == "doctor" + assert batch["protocol_version"] == constants.PROTOCOL_VERSION + + +def test_doctor_send_test_event_unauthorized_exits_2(monkeypatch, capsys): + from wildedge.transmitter import IngestResponse + + _doctor_env(monkeypatch) + monkeypatch.setattr(cli, "detect_device", _fake_device) + + class FakeTransmitter: + def __init__(self, api_key, host): + pass + + def send(self, batch): + return IngestResponse( + status="unauthorized", batch_id="", events_accepted=0, events_rejected=1 + ) + + monkeypatch.setattr(cli, "Transmitter", FakeTransmitter) + + rc = cli.main(["doctor", "--integrations", "onnx", "--send-test-event"]) + out = capsys.readouterr().out + + assert rc == 2 + assert "ingest: FAIL" in out + assert "connectivity: FAIL" in out + assert "config: PASS" in out + + +def test_doctor_send_test_event_network_error_exits_2(monkeypatch, capsys): + from wildedge.transmitter import TransmitError + + _doctor_env(monkeypatch) + monkeypatch.setattr(cli, "detect_device", _fake_device) + + class FakeTransmitter: + def __init__(self, api_key, host): + pass + + def send(self, batch): + raise TransmitError("Network error: boom") + + monkeypatch.setattr(cli, "Transmitter", FakeTransmitter) + + rc = cli.main(["doctor", "--integrations", "onnx", "--send-test-event"]) + + assert rc == 2 + assert "ingest: FAIL" in capsys.readouterr().out + + +def test_doctor_network_check_failure_exits_2(monkeypatch): + _doctor_env(monkeypatch) + monkeypatch.setattr( + cli, "network_reachability_check", lambda host: (False, "unreachable") + ) + + rc = cli.main(["doctor", "--integrations", "onnx", "--network-check"]) + + assert rc == 2 + + +def test_doctor_config_failure_wins_over_connectivity(monkeypatch): + monkeypatch.delenv(constants.ENV_DSN, raising=False) + monkeypatch.setattr(cli.importlib.util, "find_spec", lambda _: object()) + monkeypatch.setattr(cli, "check_writable_dir", lambda path: (True, str(path))) + + rc = cli.main(["doctor", "--integrations", "onnx", "--send-test-event"]) + + assert rc == 1 + + +def test_doctor_json_includes_environment_and_statuses(monkeypatch, capsys): + _doctor_env(monkeypatch) + monkeypatch.setenv(constants.ENV_INTEGRATIONS, "openai") + monkeypatch.delenv("PYTHONPATH", raising=False) + + rc = cli.main(["doctor", "--integrations", "onnx", "--format", "json"]) + data = json.loads(capsys.readouterr().out) + + assert rc == 0 + assert data["environment"]["integrations"] == "openai" + assert data["environment"]["dsn_set"] is True + assert data["environment"]["autoload_on_pythonpath"] is False + assert data["config_status"] == "PASS" + assert data["connectivity_status"] == "SKIP" diff --git a/wildedge/cli.py b/wildedge/cli.py index cbb4710..2199335 100644 --- a/wildedge/cli.py +++ b/wildedge/cli.py @@ -11,16 +11,21 @@ import socket import sys import tempfile +import uuid +from datetime import datetime, timezone from pathlib import Path from urllib.parse import urlparse from wildedge import constants +from wildedge.batch import build_batch from wildedge.client import parse_dsn +from wildedge.events.span import SpanEvent from wildedge.hubs.registry import HUBS_BY_NAME, supported_hubs from wildedge.integrations.registry import INTEGRATIONS_BY_NAME, supported_integrations from wildedge.paths import default_dead_letter_dir, default_pending_queue_dir -from wildedge.platforms import get_device_id_path +from wildedge.platforms import detect_device, get_device_id_path from wildedge.settings import read_client_env, resolve_app_identity +from wildedge.transmitter import TransmitError, Transmitter def build_parser() -> argparse.ArgumentParser: @@ -131,6 +136,11 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Attempt TCP reachability check to DSN host:port.", ) + doctor.add_argument( + "--send-test-event", + action="store_true", + help="Send one test span event to the ingest endpoint and report the result.", + ) doctor.add_argument( "--sampling-interval", type=float, @@ -327,6 +337,44 @@ def validate_runtime_config(parsed: argparse.Namespace) -> tuple[bool, str]: return True, "OK" +def ingest_round_trip(dsn: str) -> tuple[bool, str]: + """Send one minimal span event through the real pipeline; (ok, detail).""" + api_key, host_url, _ = parse_dsn(dsn) + event = SpanEvent(kind="eval", name="doctor", duration_ms=0, status="ok") + batch = build_batch( + device=detect_device(api_key, None), + models={}, + events=[event.to_dict()], + session_id=str(uuid.uuid4()), + created_at=datetime.now(timezone.utc), + ) + try: + response = Transmitter(api_key=api_key, host=host_url).send(batch) + except TransmitError as exc: + return False, str(exc) + if response.status == "accepted" and response.events_accepted >= 1: + return True, f"accepted (batch {response.batch_id})" + return False, ( + f"{response.status} (accepted={response.events_accepted}, " + f"rejected={response.events_rejected})" + ) + + +def environment_report() -> dict: + autoload_dir = str(Path(__file__).parent / "autoload") + pythonpath = os.environ.get("PYTHONPATH", "") + return { + "dsn_set": bool(os.environ.get(constants.ENV_DSN)), + "autoload_env": os.environ.get(constants.WILDEDGE_AUTOLOAD, ""), + "autoload_on_pythonpath": autoload_dir in pythonpath.split(os.pathsep), + "integrations": os.environ.get(constants.ENV_INTEGRATIONS, ""), + "hubs": os.environ.get(constants.ENV_HUBS, ""), + "strict": os.environ.get(constants.ENV_STRICT, ""), + "strict_integrations": os.environ.get(constants.ENV_STRICT_INTEGRATIONS, ""), + "propagate": os.environ.get(constants.ENV_PROPAGATE, ""), + } + + def network_reachability_check(host_url: str) -> tuple[bool, str]: parsed = urlparse(host_url) host = parsed.hostname @@ -353,6 +401,8 @@ def doctor_report(parsed: argparse.Namespace) -> dict: hubs: list[dict[str, str]] = report["hubs"] # type: ignore[assignment] ok = True + connectivity_ok: bool | None = None + report["environment"] = environment_report() client_env = read_client_env(dsn=parsed.dsn) dsn = client_env.dsn @@ -379,7 +429,19 @@ def doctor_report(parsed: argparse.Namespace) -> dict: "detail": detail, } ) - ok = ok and reachable + connectivity_ok = reachable + if parsed.send_test_event: + sent, detail = ingest_round_trip(dsn) + checks.append( + { + "name": "ingest", + "status": "OK" if sent else "FAIL", + "detail": detail, + } + ) + connectivity_ok = ( + sent if connectivity_ok is None else (connectivity_ok and sent) + ) except Exception as exc: ok = False checks.append({"name": "dsn", "status": "FAIL", "detail": str(exc)}) @@ -524,13 +586,19 @@ def doctor_report(parsed: argparse.Namespace) -> dict: else: hubs.append({"name": hub_name, "status": "OK", "detail": ""}) - report["status"] = "PASS" if ok else "FAIL" + report["config_status"] = "PASS" if ok else "FAIL" + report["connectivity_status"] = ( + "SKIP" if connectivity_ok is None else ("PASS" if connectivity_ok else "FAIL") + ) + report["status"] = "PASS" if ok and connectivity_ok is not False else "FAIL" return report def print_doctor_text(report: dict) -> None: print(f"python: {report['python']}") print(f"platform: {report['platform']}") + for key, value in report["environment"].items(): + print(f"environment[{key}]: {value}") for check in report["checks"]: name = check["name"] status = check["status"] @@ -555,16 +623,23 @@ def print_doctor_text(report: dict) -> None: print(f"hub[{name}]: {status} ({detail})") else: print(f"hub[{name}]: {status}") + print(f"config: {report['config_status']}") + print(f"connectivity: {report['connectivity_status']}") print(f"doctor: {report['status']}") def doctor(parsed: argparse.Namespace) -> int: + """Exit codes: 0 all pass, 1 config/dependency failure, 2 connectivity failure.""" report = doctor_report(parsed) if parsed.format == "json": print(json.dumps(report, sort_keys=True)) else: print_doctor_text(report) - return 0 if report["status"] == "PASS" else 1 + if report["config_status"] != "PASS": + return 1 + if report["connectivity_status"] == "FAIL": + return 2 + return 0 def main(argv: list[str] | None = None) -> int: