diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ced414c..8eda977 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -47,11 +47,19 @@ jobs: OUTPUT: /tmp/release-notes.md run: python3 scripts/build_changelog_comment.py + - name: Build llms.txt artifacts + if: github.ref_type == 'tag' + env: + OUTPUT_DIR: llms-dist + run: python3 scripts/build_llms_txt.py + - name: Create GitHub release if: github.ref_type == 'tag' uses: softprops/action-gh-release@v2 with: - files: dist/* + files: | + dist/* + llms-dist/* body_path: /tmp/release-notes.md - name: Publish package to PyPI diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..8342fc9 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,46 @@ +# Changelog + +Notable changes to wildedge-sdk. Every behavior change lands here; entries go +under Unreleased and move into a version section at release time. + +## Unreleased + +## 0.2.0 - 2026-07-14 + +### Added + +- `register_model()` accepts a `model_format` override for models without a matching extractor; `llm_api()` registers its models as format `"api"`, matching the openai/anthropic integrations instead of `"unknown"`. +- `source_from_base_url()` recognizes more provider hosts (anthropic, mistral, groq, together, deepseek, xai, google, fireworks, cerebras, perplexity, nvidia, huggingface, baseten) instead of falling back to raw hostnames, including suffix matching for per-resource subdomains (Azure OpenAI, Baseten). +- `examples/llm_api_example.py`: raw-HTTP LLM tracking with `wildedge.llm_api()`; existing examples updated to the module-level API (`wildedge.span` / `wildedge.flush` / `wildedge.register_model` instead of threading a client variable) +- Releases ship `llms.txt` and `llms-full.txt` as GitHub release assets: the full documentation for that exact version in one file, generated by `scripts/build_llms_txt.py`. README quickstart rewritten around the module-level API. +- `wildedge doctor --send-test-event`: sends one real span event through the full pipeline and reports the ingest response, proving DSN auth and connectivity end to end. The report gains an `environment` section (`WILDEDGE_*` variables, autoload PYTHONPATH status) plus `config_status` / `connectivity_status` fields. +- `wildedge.llm_api()`: provider-agnostic tracking for LLM calls made with any HTTP client (OpenRouter, vLLM, Ollama, OpenAI/Anthropic-compatible endpoints). Times the block, normalizes usage payloads from either provider shape via `call.usage()` / `call.response()`, supports TTFT marks and async use, and records exceptions as error events. See `docs/llm_api.md`. +- `register_model()` without a matching extractor defaults the model name to the explicit `model_id` instead of the placeholder object's type name. +- Process-wide default client: `wildedge.get_client()`, `wildedge.set_default_client()`, and module-level `trace`, `span`, `track_span`, `register_model`, `flush` delegating to it (#41) +- `init()` reuses the default client installed by `wildedge run` unless `dsn` is passed, so CLI and in-process init share one client (#41) +- `SpanContextManager.set_attributes()` and `fail()` for recording outcomes on an open span (#41) +- `wildedge run --strict` (env `WILDEDGE_STRICT`): exit with a reserved code instead of running untracked when bootstrap fails (120 config error, 122 internal error) +- Lazy default-client creation honors `WILDEDGE_INTEGRATIONS` and `WILDEDGE_HUBS` from the environment (#41) +- `docs/deployment.md`: the deployment contract (no-DSN behavior, strict mode, fork/exec servers, one client per process) + +### Changed + +- `wildedge doctor` exit codes are now differentiated: 0 pass, 1 configuration or dependency failure, 2 connectivity failure (`--network-check` or `--send-test-event`). A failing network check previously exited 1. +- The test suite isolates all default SDK state paths under tmp; tests no longer write to the machine-global state directory. + +- `--strict-integrations` now takes effect: a failed required integration exits the process with code 121. Previously the enforcing code path was never invoked, so the flag was silently ignored. +- `--print-startup-report` and `--no-propagate` now work under `wildedge run`; both were only wired to the unused runner path before. +- Constructing a client without a DSN logs once per process at INFO. Previously every construction logged a WARNING. + +### Removed + +- `wildedge.runtime.runner`: an alternative bootstrap entry point that `wildedge run` never invoked. `sitecustomize` is the single bootstrap path; the runner's exit codes moved to `wildedge.runtime.bootstrap` and now apply under `--strict`. + +## 0.1.5 - 2026-06-23 + +- Opt-in inference attachments (raw input/output upload) +- Fixed accelerator detection wiring; macOS CPU frequency and thermal sampling + +## 0.1.4 and earlier + +Predate this changelog; see the git history. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5de252b..baa954d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,7 +38,8 @@ 1. Fork the repository and create a feature branch off `devel`. 2. Make your changes and ensure tests pass. 3. Update documentation if needed. -4. Submit a pull request targeting `devel` with a clear description of the changes. +4. Add a `CHANGELOG.md` entry under Unreleased for any user-visible or behavioral change. +5. Submit a pull request targeting `devel` with a clear description of the changes. ## Release process diff --git a/README.md b/README.md index 4c95621..df60fd4 100644 --- a/README.md +++ b/README.md @@ -30,10 +30,12 @@ WILDEDGE_DSN="https://@ingest.wildedge.dev/" \ wildedge run --integrations timm -- python app.py ``` -Validate your environment before deploying: +Validate your environment before deploying. `--send-test-event` proves the +whole pipeline end to end by sending one span event and reporting the ingest +response (exit codes: 0 pass, 1 config failure, 2 connectivity failure): ```bash -wildedge doctor --integrations all --network-check +wildedge doctor --integrations all --send-test-event ``` Useful flags: @@ -43,7 +45,8 @@ Useful flags: | `--integrations` | Comma-separated list of integrations to activate (or `all`) | | `--hubs` | Hub trackers to activate: `huggingface`, `torchhub` | | `--print-startup-report` | Print per-integration status at startup | -| `--strict-integrations` | Fail if a requested integration can't be loaded | +| `--strict-integrations` | Exit (code 121) if a requested integration can't be instrumented | +| `--strict` | Exit (120 config, 122 internal) instead of running untracked when bootstrap fails | | `--attachments` | Enable opt-in raw input/output attachment upload | | `--no-propagate` | Don't pass WildEdge env vars to child processes | @@ -52,18 +55,23 @@ Useful flags: ```python import wildedge -client = wildedge.init( - dsn="...", # or WILDEDGE_DSN env var - integrations=["transformers"], - hubs=["huggingface"], -) +wildedge.init(integrations=["transformers"]) # optional under `wildedge run` -# models loaded after this point are tracked automatically -``` +# models loaded after this point are tracked automatically; add traces, +# spans and LLM API calls anywhere, no client instance to pass around: +with wildedge.trace(run_id="run-1"): + with wildedge.span(kind="agent_step", name="plan"): + ... -If no DSN is configured, the client becomes a no-op and logs a warning. +with wildedge.llm_api(model="openai/gpt-4o-mini", provider="openrouter") as call: + call.response(data) # LLM calls made with plain HTTP clients +``` -`init(...)` is a convenience wrapper for `WildEdge(...)` + `instrument(...)`. +One client per process: `wildedge run`, `init()`, and the module-level calls +all share it, and `init()` without `dsn` reuses whatever already exists. +Without a DSN everything is a silent no-op, so dev and CI need no +configuration. See [Deployment](https://github.com/wild-edge/wildedge-python/blob/main/docs/deployment.md) +for the full contract. ## Supported integrations **On-device** @@ -87,6 +95,10 @@ If no DSN is configured, the client becomes a no-op and logs a warning. | `anthropic` | [anthropic_example.py](https://github.com/wild-edge/wildedge-python/blob/main/examples/anthropic_example.py) | | `openai` | [openai_example.py](https://github.com/wild-edge/wildedge-python/blob/main/examples/openai_example.py) | +Calling an LLM API with a plain HTTP client instead of these libraries? Use +[`wildedge.llm_api()`](https://github.com/wild-edge/wildedge-python/blob/main/docs/llm_api.md): +[llm_api_example.py](https://github.com/wild-edge/wildedge-python/blob/main/examples/llm_api_example.py). + **Hub tracking** Pass `hubs=` to track model download provenance. Hubs are framework-agnostic and can be combined with any integration. @@ -115,6 +127,7 @@ For advanced options (batching, queue tuning, dead-letter storage, attachments), | Name | Link | |---|---| +| outfitstudio.app | https://outfitstudio.app/ | | agntr | [github.com/pmaciolek/agntr](https://github.com/pmaciolek/agntr) | | demo-app | [github.com/wild-edge/demo-app](https://github.com/wild-edge/demo-app) | | *(your project here)* | - | @@ -131,6 +144,12 @@ Report security and privacy issues to: support@wildedge.dev ## Links +- [Deployment guide](https://github.com/wild-edge/wildedge-python/blob/main/docs/deployment.md) +- [Manual tracking](https://github.com/wild-edge/wildedge-python/blob/main/docs/manual-tracking.md) +- [LLM API tracking](https://github.com/wild-edge/wildedge-python/blob/main/docs/llm_api.md) - [Compatibility Matrix](https://github.com/wild-edge/wildedge-python/blob/main/docs/compatibility.md) -- [Changelog](https://github.com/wild-edge/wildedge-python/releases) +- [Changelog](https://github.com/wild-edge/wildedge-python/blob/main/CHANGELOG.md) - [License](https://github.com/wild-edge/wildedge-python/blob/main/LICENSE) + +Each GitHub release ships `llms.txt` and `llms-full.txt`: the full +documentation for that exact version in one file, built for AI assistants. diff --git a/docs/configuration.md b/docs/configuration.md index e3cf982..2487ebe 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,6 +1,8 @@ # Configuration -Full reference for all `WildEdge` client parameters. +Full reference for all `WildEdge` client parameters. For runtime behavior +(no-DSN mode, strict mode and exit codes, forking servers), see +[Deployment](deployment.md). ## Core diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..713c016 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,122 @@ +# Deployment + +How the SDK behaves in real process models: servers that fork, workers that +exec, build steps, and processes with no DSN configured. Everything on this +page is a contract the SDK keeps, not an implementation detail. + +## How initialization happens + +There are three ways a process gets its client, in order of coverage: + +| Mode | Instrumentation coverage | Use when | +|---|---|---| +| `wildedge run -- ` | Guaranteed: patches before any user code runs | You control the start command | +| `wildedge.init(...)` at startup | Everything created after the call | You control application code | +| Lazy, from the first module-level call | Best effort: only what is created afterwards | Manual tracking only | + +All three produce the same thing: one process-wide default client that +`wildedge.span()`, `wildedge.register_model()` and the rest delegate to. +`init()` without `dsn` reuses whatever client already exists, so code written +for in-process init runs unchanged under `wildedge run`. + +The lazy mode configures itself from the environment: `WILDEDGE_DSN`, +`WILDEDGE_INTEGRATIONS`, `WILDEDGE_HUBS`. Unset integration variables mean +nothing is patched; auto-instrumentation is always opt-in. + +## The no-DSN contract + +Without `WILDEDGE_DSN`, the client is a no-op: no background threads, no +network, no patched frameworks, every event dropped. This is a supported mode +for development and CI, not an error; the SDK logs one INFO line per process +and stays quiet. Leave the SDK integrated and unset the variable wherever you +do not want telemetry (local dev, test runs, build and migration steps). + +Under `wildedge run`, a missing DSN warns on stderr and your program runs +untracked. Telemetry failing must not take production down; that is the +default policy. + +## Strict mode + +`wildedge run --strict` inverts the failure policy for deployments where +running unobserved is worse than not running. Bootstrap failures then +terminate the process before your program starts: + +| Exit code | Meaning | +|---|---| +| 120 | Configuration error (missing or invalid DSN) | +| 121 | A requested integration could not be instrumented (requires `--strict-integrations`) | +| 122 | Internal bootstrap error | + +`--strict-integrations` is its own opt-in and exits with 121 on failure even +without `--strict`; asking for it is the request to fail. + +## Forking and multi-process servers + +`wildedge run` prepends `wildedge/autoload/` to `PYTHONPATH` and replaces +itself with your command. Every Python interpreter that starts under it runs +the bundled `sitecustomize.py`, which bootstraps the runtime before any user +code. Two mechanisms make this safe across worker models: + +- Fresh interpreters (exec or spawn) bootstrap themselves via sitecustomize; + a `sys.modules` marker prevents double initialization within one + interpreter. +- Forked children inherit the parent's initialized runtime, and + `os.register_at_fork` hooks stop the SDK's background threads before each + fork and start fresh ones in parent and child afterward, so forked workers + transmit normally instead of inheriting dead threads. + +How that plays out on common servers: + +| Server | Worker model | Behavior under `wildedge run` | +|---|---|---| +| gunicorn (sync/gthread, with or without `preload_app`) | fork from master | Master bootstraps once; each worker gets fresh SDK threads via the at-fork hooks | +| uvicorn (`--workers`, `--reload`) | spawn / exec | Every worker is a fresh interpreter and bootstraps itself | +| granian | spawned worker processes | Each worker bootstraps itself | +| daphne | single process | One bootstrap, nothing special | +| waitress | single process, thread pool | One bootstrap; the client is thread-safe | +| celery (prefork pool) | fork from master | Same as gunicorn: at-fork hooks restart threads per worker | + +For managed platforms, wrap only the serving command. Build steps and +migrations run in separate processes that gain nothing from telemetry; leave +them unwrapped or unset `WILDEDGE_DSN` there. + +## One client per process + +- Framework patches are installed at most once per process; the first client + to instrument owns the patch. +- `init()` reuses the existing default client unless you pass `dsn` + explicitly, so the CLI-installed client and application code share one + client instead of racing. +- Trace and span correlation lives in contextvars at module level, not on a + client. Auto-instrumented events emitted inside `wildedge.trace(...)` / + `wildedge.span(...)` blocks correlate into the same trace no matter which + client emits them. + +## Verifying a deployment + +`wildedge doctor` answers "will events actually flow from this machine" +before you rely on it: + +```bash +wildedge doctor --integrations all --send-test-event --format json +``` + +- `--send-test-event` sends one real span event through the full pipeline + (DSN auth, ingest endpoint, batch protocol) and reports the server's + response, which a TCP-level `--network-check` cannot do. +- The report includes an `environment` section with every `WILDEDGE_*` + variable the runtime reads, plus whether the autoload dir is on + `PYTHONPATH`. +- Exit codes: 0 all pass, 1 configuration or dependency failure, 2 the + config is fine but the ingest endpoint is unreachable or rejecting. + +`--format json` makes the output machine-readable; pasting it into an issue +or an AI assistant is the intended debugging flow. + +## Environment propagation + +By default, `wildedge run` leaves its `WILDEDGE_*` variables in the +environment so that exec'd children (reload workers) can bootstrap. Pass +`--no-propagate` to have each bootstrapped process scrub the run-scoped +variables after initialization, keeping them away from nested subprocesses +you spawn yourself. diff --git a/docs/llm_api.md b/docs/llm_api.md new file mode 100644 index 0000000..7248a2c --- /dev/null +++ b/docs/llm_api.md @@ -0,0 +1,82 @@ +# Tracking LLM calls without a client library + +`wildedge.llm_api()` records LLM API calls made with any HTTP client: httpx or +requests against OpenRouter, vLLM, Ollama, or any OpenAI-compatible or +Anthropic-compatible endpoint. Use it when auto-instrumentation does not +apply because your app does not use the `openai` or `anthropic` packages. If +it does use them, prefer the integrations; they capture the same events with +zero code. + +The name is deliberate: this tracks calls to an LLM behind an *API boundary*. +An LLM running inside your own process (llama.cpp, transformers, MLX) is +covered by the [framework integrations](manual-tracking.md), which also +capture what no API boundary can expose: quantization from the model +artifact, memory footprint, load/unload timing, and hardware linkage. + +The block is timed automatically, events correlate with surrounding +`wildedge.trace()` / `wildedge.span()` blocks, and everything is a silent +no-op without a DSN. + +## Quickstart + +```python +import httpx +import wildedge + +async def generate(prompt: str, model: str) -> dict: + with wildedge.llm_api(model=model, provider="openrouter", prompt=prompt) as call: + async with httpx.AsyncClient(timeout=300) as http: + response = await http.post( + "https://openrouter.ai/api/v1/chat/completions", + headers={"Authorization": f"Bearer {API_KEY}"}, + json={"model": model, "messages": [{"role": "user", "content": prompt}]}, + ) + data = response.json() + call.response(data) + return data +``` + +`call.response()` pulls token usage, stop reason, and API metadata from the +full response payload, dict or SDK object, OpenAI shape +(`usage.prompt_tokens`, `choices[0].finish_reason`) or Anthropic shape +(`usage.input_tokens`, top-level `stop_reason`). + +Runnable version: [examples/llm_api_example.py](../examples/llm_api_example.py), +stdlib urllib against OpenRouter, no client library at all. + +## Recording pieces individually + +When you do not have a full response payload, set what you know: + +```python +with wildedge.llm_api(model="gemma-7b", base_url="http://localhost:11434") as call: + result = post_to_ollama(...) + call.usage(result["usage"]) # payload in either provider shape + call.usage(tokens_in=52, tokens_out=209) # or explicit fields; these win + call.stop_reason = "stop" + call.first_token() # TTFT mark for streaming + call.success = False # delivered but unusable output +``` + +An exception escaping the block records an error event with the exception +class as the error code, and no inference event. + +## Model identity + +Events register under `model` as the model id. `provider` names the source +directly; alternatively `base_url` derives it (`openrouter.ai` becomes +`openrouter`, unknown hosts record the hostname). Pass `prompt` or `messages` +(chat format) to attach input metadata such as prompt length. + +## Agentic pipelines + +Combine with traces for multi-step visibility: + +```python +with wildedge.trace(run_id=run_id, agent_id="skin-generator"): + for attempt in range(2): + with wildedge.span(kind="agent_step", name="generate", step_index=attempt): + with wildedge.llm_api(model=model, provider="openrouter", prompt=prompt) as call: + data = await post_chat_completion(prompt) + call.response(data) +``` diff --git a/docs/manual-tracking.md b/docs/manual-tracking.md index 8968c16..ed52a24 100644 --- a/docs/manual-tracking.md +++ b/docs/manual-tracking.md @@ -5,10 +5,47 @@ Use manual tracking when your framework is not covered by a WildEdge integration ## When to use it - Your model class is custom (e.g. a `torch.nn.Module` subclass not loaded via `timm` or `transformers`) -- You are calling a remote API not yet covered by an integration +- You are calling a remote API not yet covered by an integration (for LLM + APIs called over raw HTTP, [`wildedge.llm_api()`](llm_api.md) is the shortcut) - You want to attach input/output metadata (token counts, image dimensions, confidence scores, etc.) - You want to record user feedback tied to a specific inference +## The default client + +Every process has at most one default client, and the module-level API +delegates to it. You rarely need to hold a client instance yourself: + +```python +import wildedge + +with wildedge.trace(run_id="run-123"): + with wildedge.span(kind="agent_step", name="plan", step_index=1): + ... +``` + +The default client comes from wherever initialization happened first: + +- `wildedge run` installs one before your code loads. A later + `wildedge.init()` without `dsn` reuses it, so the same code works with or + without the CLI wrapper. +- `wildedge.init(...)` creates one (or reuses the existing default) and + registers it. +- Any module-level call (`wildedge.span`, `wildedge.register_model`, ...) + creates one lazily from the environment: `WILDEDGE_DSN` for the endpoint + (without it the client is a no-op), plus `WILDEDGE_INTEGRATIONS` and + `WILDEDGE_HUBS` for auto-instrumentation. Unset means nothing is patched. + +`wildedge.get_client()` returns the default client when you need direct +access; `wildedge.set_default_client()` overrides it (useful in tests). +`wildedge.trace` is the exception to the delegation rule: a trace only sets +correlation contextvars and emits nothing, so it never touches or creates a +client. + +Auto-instrumentation patches at creation time, so ordering matters: the CLI +patches before any user code runs, an early `init()` covers everything created +after it, and a lazily created client only covers what is created after the +first module-level call. When in doubt, call `init()` at application startup. + ## torch and keras For `torch` and `keras`, models are user-defined subclasses with no constructor to patch. Use `client.load()` to get load, unload, and inference tracking automatically: @@ -44,11 +81,12 @@ For remote APIs with no local object to inspect, pass a placeholder: ```python handle = client.register_model( - object(), + None, model_id="openai/gpt-4o", - source="https://api.openai.com", + source="openai", family="gpt-4o", version="2024-08-06", + model_format="api", ) ``` @@ -296,6 +334,30 @@ with client.trace(run_id="run-123", agent_id="agent-1"): handle.track_inference(duration_ms=t.elapsed_ms, input_modality="text", output_modality="generation") ``` +Both are also available module-level on the default client, so this is +equivalent without holding a client: + +```python +with wildedge.trace(run_id="run-123", agent_id="agent-1"): + with wildedge.span(kind="agent_step", name="plan", step_index=1): + ... +``` + +### Recording results on an open span + +The span object is mutable until the block exits; set outcomes you only know +mid-block directly on it. `set_attributes()` merges into the span's +attributes, `fail()` marks it failed with an optional summary, and an +exception escaping the block sets `status="error"` automatically: + +```python +with wildedge.span(kind="eval", name="lint") as span: + errors = lint(result) + span.set_attributes(lint_errors=len(errors)) + if errors: + span.fail(f"{len(errors)} lint errors") +``` + If you need to set correlation fields without emitting a span event, use the lower-level `span_context()` directly: diff --git a/examples/agentic_example.py b/examples/agentic_example.py index 446c812..6874184 100644 --- a/examples/agentic_example.py +++ b/examples/agentic_example.py @@ -26,7 +26,7 @@ import wildedge -we = wildedge.init( +wildedge.init( app_version="1.0.0", integrations="openai", ) @@ -96,7 +96,7 @@ def calculator(expression: str) -> str: def call_tool(name: str, arguments: dict) -> str: - with we.span( + with wildedge.span( kind="tool", name=name, input_summary=json.dumps(arguments)[:200], @@ -108,7 +108,7 @@ def call_tool(name: str, arguments: dict) -> str: def retrieve_context(query: str) -> str: """Fetch relevant context from the vector store (~120ms).""" - with we.span( + with wildedge.span( kind="retrieval", name="vector_search", input_summary=query[:200], @@ -125,7 +125,7 @@ def run_agent(task: str, step_index: int, messages: list) -> str: messages.append({"role": "user", "content": f"{task}\n\nContext: {context}"}) while True: - with we.span( + with wildedge.span( kind="agent_step", name="reason", step_index=step_index, @@ -172,10 +172,10 @@ def run_agent(task: str, step_index: int, messages: list) -> str: system_prompt = "You are a helpful assistant. Use tools when needed." messages = [{"role": "system", "content": system_prompt}] -with we.trace(agent_id="demo-agent", run_id=str(uuid.uuid4())): +with wildedge.trace(agent_id="demo-agent", run_id=str(uuid.uuid4())): for i, task in enumerate(TASKS, start=1): print(f"\nTask {i}: {task}") reply = run_agent(task, step_index=i, messages=messages) print(f"Reply: {reply}") -we.flush() +wildedge.flush() diff --git a/examples/anthropic_example.py b/examples/anthropic_example.py index 62fca35..eab152c 100644 --- a/examples/anthropic_example.py +++ b/examples/anthropic_example.py @@ -18,7 +18,7 @@ import wildedge -client = wildedge.init( +wildedge.init( app_version="1.0.0", # uses WILDEDGE_DSN if set; otherwise no-op integrations="anthropic", ) @@ -44,5 +44,5 @@ print(event.delta.text, end="", flush=True) print("\n") -client.flush() +wildedge.flush() print("Done. Events flushed to WildEdge.") diff --git a/examples/attachments_example.py b/examples/attachments_example.py index 4eb0735..4669561 100644 --- a/examples/attachments_example.py +++ b/examples/attachments_example.py @@ -26,7 +26,7 @@ def redact(attachments: list[Attachment]) -> list[Attachment]: return [a for a in attachments if a.content_type != "application/secret"] -client = wildedge.init( +wildedge.init( app_version="1.0.0", attachments_enabled=True, max_attachments_per_inference=5, @@ -35,7 +35,7 @@ def redact(attachments: list[Attachment]) -> list[Attachment]: attachment_filter=redact, ) -handle = client.register_model( +handle = wildedge.register_model( object(), model_id="doc-classifier-v1", source="local", @@ -59,4 +59,4 @@ def redact(attachments: list[Attachment]) -> list[Attachment]: print(f"tracked inference {inference_id[:8]}… with 2 attachments") # Bytes upload in the background; flush/close lets buffered events drain. -client.close() +wildedge.flush() diff --git a/examples/feedback_example.py b/examples/feedback_example.py index ff06292..a1896dd 100644 --- a/examples/feedback_example.py +++ b/examples/feedback_example.py @@ -21,14 +21,14 @@ CONFIDENCE_THRESHOLD = 0.6 -client = wildedge.init( +wildedge.init( app_version="1.0.0", # uses WILDEDGE_DSN if set; otherwise no-op integrations="timm", ) model = timm.create_model("resnet18", pretrained=True) model.eval() -handle = client.register_model( +handle = wildedge.register_model( model ) # auto-instrumented already; returns existing handle diff --git a/examples/gguf_example.py b/examples/gguf_example.py index bd50f98..861603d 100644 --- a/examples/gguf_example.py +++ b/examples/gguf_example.py @@ -12,7 +12,7 @@ import wildedge -client = wildedge.init( +wildedge.init( app_version="1.0.0", # uses WILDEDGE_DSN if set; otherwise no-op integrations="gguf", hubs=["huggingface"], diff --git a/examples/llm_api_example.py b/examples/llm_api_example.py new file mode 100644 index 0000000..1afe2ff --- /dev/null +++ b/examples/llm_api_example.py @@ -0,0 +1,63 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = ["wildedge-sdk"] +# +# [tool.uv.sources] +# wildedge-sdk = { path = "..", editable = true } +# /// +"""Track LLM calls made over plain HTTP with wildedge.llm_api(). + +No openai or anthropic client library involved: requests go through stdlib +urllib against OpenRouter's OpenAI-compatible endpoint, and llm_api() records +the same inference events the integrations emit (tokens, TTFT, stop reason), +correlated into the surrounding trace and spans. + +Run with: uv run llm_api_example.py +Requires: OPENROUTER_API_KEY environment variable. Set WILDEDGE_DSN to send events. +""" + +import json +import os +import urllib.request +import uuid + +import wildedge + +wildedge.init(app_version="1.0.0") # uses WILDEDGE_DSN if set; otherwise no-op + +URL = "https://openrouter.ai/api/v1/chat/completions" +MODEL = "openai/gpt-4o-mini" + + +def chat(prompt: str) -> dict: + request = urllib.request.Request( + URL, + data=json.dumps( + {"model": MODEL, "messages": [{"role": "user", "content": prompt}]} + ).encode(), + headers={ + "Authorization": f"Bearer {os.getenv('OPENROUTER_API_KEY')}", + "Content-Type": "application/json", + }, + ) + with urllib.request.urlopen(request, timeout=120) as response: + return json.load(response) + + +PROMPTS = [ + "What is on-device AI in one sentence?", + "Name three edge inference runtimes.", +] + +with wildedge.trace(agent_id="llm-api-example", run_id=str(uuid.uuid4())): + for step, prompt in enumerate(PROMPTS): + with wildedge.span(kind="agent_step", name="ask", step_index=step): + with wildedge.llm_api( + model=MODEL, provider="openrouter", prompt=prompt + ) as call: + data = chat(prompt) + call.response(data) + print(f"Q: {prompt}\nA: {data['choices'][0]['message']['content']}\n") + +wildedge.flush() +print("Done. Events flushed to WildEdge.") diff --git a/examples/mlx_example.py b/examples/mlx_example.py index 61906e0..e7c0068 100644 --- a/examples/mlx_example.py +++ b/examples/mlx_example.py @@ -52,7 +52,7 @@ def main() -> None: # init() constructs the client and instruments mlx; must be called # before any model is loaded. - client = wildedge.init( + wildedge.init( app_version="1.0.0", # uses WILDEDGE_DSN if set; otherwise no-op integrations="mlx", hubs=["huggingface"], @@ -73,7 +73,7 @@ def main() -> None: print(f"[{i}] Q: {prompt}") print(f" A: {response}\n") - client.flush() + wildedge.flush() print("Done. Events flushed to WildEdge.") diff --git a/examples/onnx_example.py b/examples/onnx_example.py index e7a6f78..eb91f09 100644 --- a/examples/onnx_example.py +++ b/examples/onnx_example.py @@ -13,7 +13,7 @@ import wildedge -client = wildedge.init( +wildedge.init( app_version="1.0.0", # uses WILDEDGE_DSN if set; otherwise no-op integrations="onnx", hubs=["huggingface"], diff --git a/examples/openai_example.py b/examples/openai_example.py index ad2d3a9..5179b82 100644 --- a/examples/openai_example.py +++ b/examples/openai_example.py @@ -18,7 +18,7 @@ import wildedge -client = wildedge.init( +wildedge.init( app_version="1.0.0", # uses WILDEDGE_DSN if set; otherwise no-op integrations="openai", ) @@ -46,5 +46,5 @@ print(chunk.choices[0].delta.content, end="", flush=True) print("\n") -client.flush() +wildedge.flush() print("Done. Events flushed to WildEdge.") diff --git a/examples/tensorflow_example.py b/examples/tensorflow_example.py index 3cf1962..cca72a8 100644 --- a/examples/tensorflow_example.py +++ b/examples/tensorflow_example.py @@ -17,7 +17,7 @@ import wildedge -client = wildedge.init( +wildedge.init( app_version="1.0.0", # uses WILDEDGE_DSN if set; otherwise no-op integrations="tensorflow", ) @@ -48,4 +48,4 @@ def build_and_save_model(save_path: Path) -> None: print("output shape:", tuple(output.shape)) -client.close() +wildedge.flush() diff --git a/examples/timm_example.py b/examples/timm_example.py index 5b4ffa1..5930368 100644 --- a/examples/timm_example.py +++ b/examples/timm_example.py @@ -19,7 +19,7 @@ import wildedge -client = wildedge.init( +wildedge.init( app_version="1.0.0", # uses WILDEDGE_DSN if set; otherwise no-op integrations="timm", hubs=["huggingface", "torchhub"], diff --git a/examples/transformers_example.py b/examples/transformers_example.py index 92e8a1d..4c0e53a 100644 --- a/examples/transformers_example.py +++ b/examples/transformers_example.py @@ -92,7 +92,7 @@ def main() -> None: ) args = parser.parse_args() - client = wildedge.init( + wildedge.init( app_version="1.0.0", # uses WILDEDGE_DSN if set; otherwise no-op integrations="transformers", hubs=["huggingface"], @@ -103,7 +103,7 @@ def main() -> None: args.task ]() - client.flush() + wildedge.flush() print("\nDone. Events flushed to WildEdge.") diff --git a/pyproject.toml b/pyproject.toml index 197d73c..cc84401 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "wildedge-sdk" -version = "0.1.5" +version = "0.2.0" description = "ML inference monitoring for Python: on-device and remote models" readme = "README.md" requires-python = ">=3.10" diff --git a/scripts/build_llms_txt.py b/scripts/build_llms_txt.py new file mode 100644 index 0000000..5991ed6 --- /dev/null +++ b/scripts/build_llms_txt.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Build llms.txt (index) and llms-full.txt (full content) from README + docs. + +llms.txt is a link index per https://llmstxt.org; llms-full.txt is the entire +documentation concatenated and stamped with the release version, so an agent +gets correct, versioned docs in one fetch. Run at release time; outputs land +in OUTPUT_DIR (default: llms-dist). +""" + +from __future__ import annotations + +import os +import re +from datetime import datetime, timezone +from pathlib import Path + +REPO_ROOT = Path(__file__).parent.parent + +# Order is the reading order in llms-full.txt. +SOURCES: list[tuple[str, str, str]] = [ + ("README.md", "", "WildEdge Python SDK"), + ("docs/configuration.md", "configuration", "Configuration"), + ("docs/deployment.md", "deployment", "Deployment"), + ("docs/manual-tracking.md", "manual-tracking", "Manual tracking"), + ("docs/llm_api.md", "llm_api", "LLM API tracking"), + ("docs/compatibility.md", "compatibility", "Compatibility matrix"), +] + + +def project_version() -> str: + # Regex instead of tomllib: the script must run on 3.10, where tomllib + # does not exist, and this is the only field it needs. + text = (REPO_ROOT / "pyproject.toml").read_text() + match = re.search(r'^version = "([^"]+)"', text, re.MULTILINE) + if match is None: + raise SystemExit("could not find project version in pyproject.toml") + return match.group(1) + + +def clean_markdown(text: str) -> str: + """Drop HTML tags and badge lines; they are noise to a text consumer.""" + kept = [ + line for line in text.splitlines() if not line.lstrip().startswith(("<", "[![")) + ] + return "\n".join(kept).strip() + "\n" + + +def first_paragraph(text: str) -> str: + """First prose paragraph outside code fences, as the index description.""" + collected: list[str] = [] + in_fence = False + for line in clean_markdown(text).splitlines(): + stripped = line.strip() + if stripped.startswith("```"): + in_fence = not in_fence + continue + if in_fence: + continue + if stripped.startswith(("#", ">", "|", "-")): + if collected: + break + continue + if not stripped: + if collected: + break + continue + collected.append(stripped) + paragraph = " ".join(collected) + return paragraph if len(paragraph) <= 220 else paragraph[:217] + "..." + + +def build(output_dir: Path, base_url: str) -> None: + version = project_version() + stamp = ( + f"wildedge-sdk {version} documentation, generated " + f"{datetime.now(timezone.utc).date().isoformat()} from the v{version} release." + ) + + index_lines = [ + "# WildEdge Python SDK", + "", + f"> {stamp}", + "", + "On-device ML inference monitoring for Python.", + "", + "## Docs", + "", + ] + full_parts = [f"# WildEdge Python SDK ({version})\n\n> {stamp}\n"] + + for relpath, slug, title in SOURCES: + path = REPO_ROOT / relpath + text = path.read_text(encoding="utf-8") + url = f"{base_url}/{slug}" if slug else base_url + description = first_paragraph(text) + index_lines.append(f"- [{title}]({url}): {description}") + full_parts.append(f"\n---\n\n{clean_markdown(text)}") + + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "llms.txt").write_text( + "\n".join(index_lines) + "\n", encoding="utf-8" + ) + (output_dir / "llms-full.txt").write_text("".join(full_parts), encoding="utf-8") + print(f"wrote {output_dir / 'llms.txt'} and {output_dir / 'llms-full.txt'}") + + +def main() -> None: + output_dir = Path(os.environ.get("OUTPUT_DIR", "llms-dist")) + base_url = os.environ.get("BASE_URL", "https://docs.wildedge.dev").rstrip("/") + build(output_dir, base_url) + + +if __name__ == "__main__": + main() diff --git a/scripts/check_tag_version.py b/scripts/check_tag_version.py index 08169f5..b20ff7a 100644 --- a/scripts/check_tag_version.py +++ b/scripts/check_tag_version.py @@ -4,10 +4,9 @@ from __future__ import annotations import os +import re from pathlib import Path -import tomllib - def main() -> None: tag = os.environ["TAG_NAME"] @@ -15,8 +14,13 @@ def main() -> None: raise SystemExit(f"Expected tag to start with 'v', got: {tag}") tag_version = tag[1:] - data = tomllib.loads(Path("pyproject.toml").read_text()) - project_version = data["project"]["version"] + # Regex instead of tomllib so the script also runs on Python 3.10. + match = re.search( + r'^version = "([^"]+)"', Path("pyproject.toml").read_text(), re.MULTILINE + ) + if match is None: + raise SystemExit("could not find project version in pyproject.toml") + project_version = match.group(1) if tag_version != project_version: raise SystemExit( diff --git a/tests/conftest.py b/tests/conftest.py index e8c8199..3c8890a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,6 +19,30 @@ def reset_hardware_sampler(): stop_sampler() +@pytest.fixture(autouse=True) +def reset_default_client(): + from wildedge.defaults import set_default_client + + set_default_client(None) + yield + set_default_client(None) + + +@pytest.fixture(autouse=True) +def isolate_sdk_state(tmp_path, monkeypatch): + """Keep every default persistence path under tmp. + + Without this, tests that construct real clients write queues, registries + and dead letters to the machine-global SDK state dir, leaking state + between runs and machines (and once masking a real regression locally + while CI failed). + """ + import wildedge.paths as paths + + monkeypatch.setattr(paths, "default_sdk_state_dir", lambda: tmp_path / "state") + monkeypatch.setattr(paths, "default_sdk_cache_dir", lambda: tmp_path / "cache") + + PLATFORM_MARKS = { "requires_linux": "linux", "requires_macos": "darwin", diff --git a/tests/test_autoload.py b/tests/test_autoload.py index d16c04a..578af6a 100644 --- a/tests/test_autoload.py +++ b/tests/test_autoload.py @@ -176,3 +176,153 @@ def test_uvicorn_reload_worker_bootstraps(tmp_path): ) assert result.returncode == 0, result.stderr assert "installed: True" in result.stdout + + +def test_strict_missing_dsn_exits_120(monkeypatch): + from wildedge.runtime.bootstrap import RuntimeConfigError + + monkeypatch.delitem(sys.modules, _MARKER, raising=False) + monkeypatch.setenv("WILDEDGE_AUTOLOAD", "1") + monkeypatch.setenv("WILDEDGE_STRICT", "1") + monkeypatch.delenv("WILDEDGE_DSN", raising=False) + + with ( + patch( + "wildedge.runtime.bootstrap.install_runtime", + side_effect=RuntimeConfigError("WILDEDGE_DSN must be set"), + ), + patch("os._exit") as fake_exit, + ): + _reload_sitecustomize() + + fake_exit.assert_called_once_with(120) + + +def test_strict_integration_error_exits_121_without_strict_env(monkeypatch): + """--strict-integrations is its own opt-in; it exits even without --strict.""" + from wildedge.runtime.bootstrap import RuntimeStrictIntegrationError + + monkeypatch.delitem(sys.modules, _MARKER, raising=False) + monkeypatch.setenv("WILDEDGE_AUTOLOAD", "1") + monkeypatch.delenv("WILDEDGE_STRICT", raising=False) + + with ( + patch( + "wildedge.runtime.bootstrap.install_runtime", + side_effect=RuntimeStrictIntegrationError("strict fail"), + ), + patch("os._exit") as fake_exit, + ): + _reload_sitecustomize() + + fake_exit.assert_called_once_with(121) + + +def test_strict_internal_error_exits_122(monkeypatch): + monkeypatch.delitem(sys.modules, _MARKER, raising=False) + monkeypatch.setenv("WILDEDGE_AUTOLOAD", "1") + monkeypatch.setenv("WILDEDGE_STRICT", "1") + + with ( + patch( + "wildedge.runtime.bootstrap.install_runtime", + side_effect=RuntimeError("boom"), + ), + patch("os._exit") as fake_exit, + ): + _reload_sitecustomize() + + fake_exit.assert_called_once_with(122) + + +def test_non_strict_config_error_warns_and_continues(monkeypatch, capsys): + from wildedge.runtime.bootstrap import RuntimeConfigError + + monkeypatch.delitem(sys.modules, _MARKER, raising=False) + monkeypatch.setenv("WILDEDGE_AUTOLOAD", "1") + monkeypatch.delenv("WILDEDGE_STRICT", raising=False) + + with patch( + "wildedge.runtime.bootstrap.install_runtime", + side_effect=RuntimeConfigError("no dsn"), + ): + _reload_sitecustomize() # must not raise + + assert "running without telemetry" in capsys.readouterr().err + + +def test_startup_report_printed_from_env(monkeypatch, capsys): + class FakeContext: + debug = False + print_startup_report = False + integration_statuses: list = [] + + monkeypatch.delitem(sys.modules, _MARKER, raising=False) + monkeypatch.setenv("WILDEDGE_AUTOLOAD", "1") + monkeypatch.setenv("WILDEDGE_PRINT_STARTUP_REPORT", "1") + + with ( + patch( + "wildedge.runtime.bootstrap.install_runtime", + return_value=FakeContext(), + ), + patch( + "wildedge.runtime.bootstrap.format_startup_report", + return_value="report-text", + ), + ): + _reload_sitecustomize() + + assert "report-text" in capsys.readouterr().err + + +def test_no_propagate_clears_runtime_env(monkeypatch): + import os + + class FakeContext: + debug = False + print_startup_report = False + + monkeypatch.delitem(sys.modules, _MARKER, raising=False) + monkeypatch.setenv("WILDEDGE_AUTOLOAD", "1") + monkeypatch.setenv("WILDEDGE_PROPAGATE", "0") + monkeypatch.setenv("WILDEDGE_INTEGRATIONS", "all") + + with patch( + "wildedge.runtime.bootstrap.install_runtime", + return_value=FakeContext(), + ): + _reload_sitecustomize() + + assert "WILDEDGE_AUTOLOAD" not in os.environ + assert "WILDEDGE_INTEGRATIONS" not in os.environ + + +def test_strict_missing_dsn_exit_code_in_real_interpreter(tmp_path): + """sys.exit inside sitecustomize must terminate interpreter startup with our code.""" + import os + import pathlib + + autoload_dir = str( + pathlib.Path( + importlib.util.find_spec("wildedge.autoload.sitecustomize").origin + ).parent + ) + + env = os.environ.copy() + env["WILDEDGE_AUTOLOAD"] = "1" + env["WILDEDGE_STRICT"] = "1" + env.pop("WILDEDGE_DSN", None) + pythonpath = env.get("PYTHONPATH", "") + env["PYTHONPATH"] = autoload_dir + (os.pathsep + pythonpath if pythonpath else "") + + result = subprocess.run( + [sys.executable, "-c", "print('should not run')"], + env=env, + capture_output=True, + text=True, + ) + + assert result.returncode == 120, (result.returncode, result.stderr) + assert "should not run" not in result.stdout + assert "WILDEDGE_DSN must be set" in result.stderr diff --git a/tests/test_build_llms_txt.py b/tests/test_build_llms_txt.py new file mode 100644 index 0000000..23c9800 --- /dev/null +++ b/tests/test_build_llms_txt.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + +SCRIPT = Path(__file__).parent.parent / "scripts" / "build_llms_txt.py" + + +def load_script(): + spec = importlib.util.spec_from_file_location("build_llms_txt", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_build_produces_index_and_full_text(tmp_path): + module = load_script() + module.build(tmp_path, "https://docs.wildedge.dev") + + index = (tmp_path / "llms.txt").read_text() + full = (tmp_path / "llms-full.txt").read_text() + version = module.project_version() + + assert index.startswith("# WildEdge Python SDK") + assert f"wildedge-sdk {version} documentation" in index + assert "- [WildEdge Python SDK](https://docs.wildedge.dev):" in index + for slug in ("configuration", "deployment", "manual-tracking", "llm_api"): + assert f"https://docs.wildedge.dev/{slug}" in index + + assert f"# WildEdge Python SDK ({version})" in full + # One recognizable phrase per source document proves each was included. + assert "Full reference for all `WildEdge` client parameters" in full + assert "pre-deploy" in full or "wildedge run" in full + assert "register_model" in full + assert "llm_api" in full + + +def test_clean_markdown_strips_html_and_badges(): + module = load_script() + text = '

logo

\n[![CI](x)](y)\n# Title\nprose line\n' + cleaned = module.clean_markdown(text) + assert " None: + # sys.exit cannot be used during sitecustomize import: site.py reports the + # SystemExit as a fatal startup error with a traceback and exit code 1. + sys.stderr.flush() + os._exit(code) + + if _INSTALLED_MARKER not in sys.modules: # Activate if the CLI launched this process, or if WILDEDGE_DSN is set # and the user has manually prepended wildedge/autoload/ to PYTHONPATH. @@ -29,11 +44,53 @@ # Set the guard before importing wildedge to prevent re-entry. sys.modules[_INSTALLED_MARKER] = True # type: ignore[assignment] try: - from wildedge.runtime.bootstrap import install_runtime # noqa: PLC0415 + from wildedge.runtime.bootstrap import ( # noqa: PLC0415 + EXIT_BOOTSTRAP_INTERNAL_ERROR, + EXIT_CONFIG_ERROR, + EXIT_STRICT_INTEGRATION_ERROR, + RuntimeConfigError, + RuntimeStrictIntegrationError, + clear_runtime_env, + format_startup_report, + install_runtime, + ) + from wildedge.settings import read_runner_env # noqa: PLC0415 + + _runner_env = read_runner_env() + _context = None + try: + # Don't install signal handlers: the host process (gunicorn, + # celery, etc.) manages SIGTERM/SIGINT itself. + _context = install_runtime(install_signal_handlers=False) + except RuntimeStrictIntegrationError as exc: + # Only raised when --strict-integrations was requested. + print(f"wildedge: {exc}", file=sys.stderr) + _die(EXIT_STRICT_INTEGRATION_ERROR) + except RuntimeConfigError as exc: + if _runner_env.strict: + print(f"wildedge: {exc}", file=sys.stderr) + _die(EXIT_CONFIG_ERROR) + print( + f"wildedge: {exc}; running without telemetry", + file=sys.stderr, + ) + except Exception as exc: + if _runner_env.strict: + print( + f"wildedge: bootstrap internal error: {exc}", + file=sys.stderr, + ) + _die(EXIT_BOOTSTRAP_INTERNAL_ERROR) + print(f"wildedge: bootstrap failed: {exc}", file=sys.stderr) - # Don't install signal handlers: the host process (gunicorn, celery, - # etc.) manages SIGTERM/SIGINT itself. - install_runtime(install_signal_handlers=False) + if _context is not None and ( + getattr(_context, "debug", False) + or getattr(_context, "print_startup_report", False) + or _runner_env.print_startup_report + ): + print(format_startup_report(_context), file=sys.stderr) + if not _runner_env.propagate: + clear_runtime_env() except Exception as exc: # pragma: no cover print(f"wildedge: bootstrap failed: {exc}", file=sys.stderr) diff --git a/wildedge/cli.py b/wildedge/cli.py index 967b23b..2199335 100644 --- a/wildedge/cli.py +++ b/wildedge/cli.py @@ -11,16 +11,21 @@ import socket import sys import tempfile +import uuid +from datetime import datetime, timezone from pathlib import Path from urllib.parse import urlparse from wildedge import constants +from wildedge.batch import build_batch from wildedge.client import parse_dsn +from wildedge.events.span import SpanEvent from wildedge.hubs.registry import HUBS_BY_NAME, supported_hubs from wildedge.integrations.registry import INTEGRATIONS_BY_NAME, supported_integrations from wildedge.paths import default_dead_letter_dir, default_pending_queue_dir -from wildedge.platforms import get_device_id_path +from wildedge.platforms import detect_device, get_device_id_path from wildedge.settings import read_client_env, resolve_app_identity +from wildedge.transmitter import TransmitError, Transmitter def build_parser() -> argparse.ArgumentParser: @@ -78,6 +83,14 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Fail startup if any requested integration cannot be instrumented.", ) + run.add_argument( + "--strict", + action="store_true", + help=( + "Exit instead of running untracked when bootstrap fails " + "(120 config error, 122 internal error). Default: warn and run." + ), + ) propagation = run.add_mutually_exclusive_group() propagation.add_argument( "--propagate", @@ -123,6 +136,11 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Attempt TCP reachability check to DSN host:port.", ) + doctor.add_argument( + "--send-test-event", + action="store_true", + help="Send one test span event to the ingest endpoint and report the result.", + ) doctor.add_argument( "--sampling-interval", type=float, @@ -251,6 +269,7 @@ def run_command(parsed: argparse.Namespace) -> int: env[constants.ENV_ATTACHMENTS_ENABLED] = "1" env[constants.ENV_PROPAGATE] = "1" if parsed.propagate else "0" env[constants.ENV_STRICT_INTEGRATIONS] = "1" if parsed.strict_integrations else "0" + env[constants.ENV_STRICT] = "1" if parsed.strict else "0" env[constants.ENV_PRINT_STARTUP_REPORT] = ( "1" if parsed.print_startup_report else "0" ) @@ -318,6 +337,44 @@ def validate_runtime_config(parsed: argparse.Namespace) -> tuple[bool, str]: return True, "OK" +def ingest_round_trip(dsn: str) -> tuple[bool, str]: + """Send one minimal span event through the real pipeline; (ok, detail).""" + api_key, host_url, _ = parse_dsn(dsn) + event = SpanEvent(kind="eval", name="doctor", duration_ms=0, status="ok") + batch = build_batch( + device=detect_device(api_key, None), + models={}, + events=[event.to_dict()], + session_id=str(uuid.uuid4()), + created_at=datetime.now(timezone.utc), + ) + try: + response = Transmitter(api_key=api_key, host=host_url).send(batch) + except TransmitError as exc: + return False, str(exc) + if response.status == "accepted" and response.events_accepted >= 1: + return True, f"accepted (batch {response.batch_id})" + return False, ( + f"{response.status} (accepted={response.events_accepted}, " + f"rejected={response.events_rejected})" + ) + + +def environment_report() -> dict: + autoload_dir = str(Path(__file__).parent / "autoload") + pythonpath = os.environ.get("PYTHONPATH", "") + return { + "dsn_set": bool(os.environ.get(constants.ENV_DSN)), + "autoload_env": os.environ.get(constants.WILDEDGE_AUTOLOAD, ""), + "autoload_on_pythonpath": autoload_dir in pythonpath.split(os.pathsep), + "integrations": os.environ.get(constants.ENV_INTEGRATIONS, ""), + "hubs": os.environ.get(constants.ENV_HUBS, ""), + "strict": os.environ.get(constants.ENV_STRICT, ""), + "strict_integrations": os.environ.get(constants.ENV_STRICT_INTEGRATIONS, ""), + "propagate": os.environ.get(constants.ENV_PROPAGATE, ""), + } + + def network_reachability_check(host_url: str) -> tuple[bool, str]: parsed = urlparse(host_url) host = parsed.hostname @@ -344,6 +401,8 @@ def doctor_report(parsed: argparse.Namespace) -> dict: hubs: list[dict[str, str]] = report["hubs"] # type: ignore[assignment] ok = True + connectivity_ok: bool | None = None + report["environment"] = environment_report() client_env = read_client_env(dsn=parsed.dsn) dsn = client_env.dsn @@ -370,7 +429,19 @@ def doctor_report(parsed: argparse.Namespace) -> dict: "detail": detail, } ) - ok = ok and reachable + connectivity_ok = reachable + if parsed.send_test_event: + sent, detail = ingest_round_trip(dsn) + checks.append( + { + "name": "ingest", + "status": "OK" if sent else "FAIL", + "detail": detail, + } + ) + connectivity_ok = ( + sent if connectivity_ok is None else (connectivity_ok and sent) + ) except Exception as exc: ok = False checks.append({"name": "dsn", "status": "FAIL", "detail": str(exc)}) @@ -515,13 +586,19 @@ def doctor_report(parsed: argparse.Namespace) -> dict: else: hubs.append({"name": hub_name, "status": "OK", "detail": ""}) - report["status"] = "PASS" if ok else "FAIL" + report["config_status"] = "PASS" if ok else "FAIL" + report["connectivity_status"] = ( + "SKIP" if connectivity_ok is None else ("PASS" if connectivity_ok else "FAIL") + ) + report["status"] = "PASS" if ok and connectivity_ok is not False else "FAIL" return report def print_doctor_text(report: dict) -> None: print(f"python: {report['python']}") print(f"platform: {report['platform']}") + for key, value in report["environment"].items(): + print(f"environment[{key}]: {value}") for check in report["checks"]: name = check["name"] status = check["status"] @@ -546,16 +623,23 @@ def print_doctor_text(report: dict) -> None: print(f"hub[{name}]: {status} ({detail})") else: print(f"hub[{name}]: {status}") + print(f"config: {report['config_status']}") + print(f"connectivity: {report['connectivity_status']}") print(f"doctor: {report['status']}") def doctor(parsed: argparse.Namespace) -> int: + """Exit codes: 0 all pass, 1 config/dependency failure, 2 connectivity failure.""" report = doctor_report(parsed) if parsed.format == "json": print(json.dumps(report, sort_keys=True)) else: print_doctor_text(report) - return 0 if report["status"] == "PASS" else 1 + if report["config_status"] != "PASS": + return 1 + if report["connectivity_status"] == "FAIL": + return 2 + return 0 def main(argv: list[str] | None = None) -> int: diff --git a/wildedge/client.py b/wildedge/client.py index 48fbfe2..8787664 100644 --- a/wildedge/client.py +++ b/wildedge/client.py @@ -109,6 +109,9 @@ def parse_dsn(dsn: str) -> tuple[str, str, str]: return parsed.username, host, project_key +# One informative log line per process for the intentional no-DSN mode. +_noop_notice_logged = False + DEFAULT_EXTRACTORS: list[BaseExtractor] = [ OnnxExtractor(), GgufExtractor(), @@ -216,6 +219,18 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc_val, exc_tb): return self.__exit__(exc_type, exc_val, exc_tb) + def set_attributes(self, **attributes: Any) -> None: + """Merge ``attributes`` into the span's attribute dict.""" + if self.attributes is None: + self.attributes = {} + self.attributes.update(attributes) + + def fail(self, detail: str | None = None) -> None: + """Mark the span as failed; ``detail`` becomes the output summary.""" + self.status = "error" + if detail is not None: + self.output_summary = detail + class WildEdge: """ @@ -431,12 +446,20 @@ def __init__( logger.debug("wildedge: client initialized (session=%s)", self.session_id) def _init_noop(self, *, debug: bool, device: DeviceInfo | None) -> None: + global _noop_notice_logged self.noop = True self.debug = debug self.closed = True - logger.warning( + # Running without a DSN is a supported mode (dev, CI), not a fault: + # say so once per process at info, then stay quiet. + message = ( "wildedge: no DSN configured; client is disabled (events will be dropped)" ) + if _noop_notice_logged: + logger.debug(message) + else: + logger.info(message) + _noop_notice_logged = True self.api_key = None self.device = device self.session_id = str(uuid.uuid4()) @@ -479,13 +502,15 @@ def register_model( family: str | None = None, version: str | None = None, quantization: str | None = None, + model_format: str | None = None, auto_instrument: bool = True, ) -> ModelHandle: """ Register a model and return a handle for tracking events. Auto-extracts metadata from ONNX Runtime and GGUF/llama.cpp objects. - User-supplied kwargs override extracted values. + User-supplied kwargs override extracted values. ``model_format`` + applies when no extractor matches (e.g. ``"api"`` for remote models). """ overrides = { k: v @@ -495,6 +520,7 @@ def register_model( "family": family, "version": version, "quantization": quantization, + "format": model_format, }.items() if v is not None } @@ -506,8 +532,10 @@ def register_model( else: # No extractor matched - require explicit id model_id = overrides.pop("id", None) - model_name = overrides.pop("model_name", None) or ( - str(type(model_obj).__name__) + model_name = ( + overrides.pop("model_name", None) + or model_id + or str(type(model_obj).__name__) ) info = ModelInfo( model_name=model_name, diff --git a/wildedge/constants.py b/wildedge/constants.py index 87b5afd..98776fa 100644 --- a/wildedge/constants.py +++ b/wildedge/constants.py @@ -16,6 +16,7 @@ ENV_INTEGRATIONS = "WILDEDGE_INTEGRATIONS" ENV_HUBS = "WILDEDGE_HUBS" ENV_STRICT_INTEGRATIONS = "WILDEDGE_STRICT_INTEGRATIONS" +ENV_STRICT = "WILDEDGE_STRICT" ENV_PROPAGATE = "WILDEDGE_PROPAGATE" ENV_PRINT_STARTUP_REPORT = "WILDEDGE_PRINT_STARTUP_REPORT" ENV_SAMPLING_INTERVAL = "WILDEDGE_SAMPLING_INTERVAL_S" diff --git a/wildedge/convenience.py b/wildedge/convenience.py index a9e61aa..bfa2f1a 100644 --- a/wildedge/convenience.py +++ b/wildedge/convenience.py @@ -4,6 +4,7 @@ from typing import Any from wildedge.client import WildEdge +from wildedge.defaults import peek_default_client, set_default_client from wildedge.logging import logger @@ -22,11 +23,26 @@ def init( **kwargs: Any, ) -> WildEdge: """ - Convenience initializer: construct a WildEdge client and instrument integrations. + Initialize the process default client and instrument integrations. - Additional keyword arguments are forwarded to WildEdge(...). + When a default client already exists (typically installed by ``wildedge + run`` before user code loaded) and no ``dsn`` is passed, that client is + reused: the requested integrations are applied to it (instrumenting is + idempotent per process) and it is returned. Passing ``dsn`` always + constructs a fresh client and makes it the new default. + + Additional keyword arguments are forwarded to WildEdge(...) when a new + client is constructed. """ - client = WildEdge(**kwargs) + client = None if "dsn" in kwargs else peek_default_client() + if client is not None and kwargs: + logger.warning( + "wildedge: init() reusing the existing default client; ignoring %s", + ", ".join(sorted(kwargs)), + ) + if client is None: + client = WildEdge(**kwargs) + normalized_integrations = _normalize_list(integrations) normalized_hubs = _normalize_list(hubs) @@ -38,4 +54,5 @@ def init( elif getattr(client, "debug", False): logger.debug("wildedge: init called without integrations or hubs") + set_default_client(client) return client diff --git a/wildedge/defaults.py b/wildedge/defaults.py new file mode 100644 index 0000000..1a29dda --- /dev/null +++ b/wildedge/defaults.py @@ -0,0 +1,198 @@ +"""Process-wide default client and the module-level tracking API. + +One client per process is the common case: `wildedge run` installs one before +user code loads, `wildedge.init()` creates or reuses one, and the module-level +functions below (`wildedge.trace`, `wildedge.span`, ...) delegate to whichever +client is current. Application code never has to thread a client instance +through call sites. +""" + +from __future__ import annotations + +import os +import threading +from typing import Any + +from wildedge import constants +from wildedge.client import SpanContextManager, WildEdge +from wildedge.events.span import SpanKind, SpanStatus +from wildedge.hubs.registry import supported_hubs +from wildedge.integrations.registry import supported_integrations +from wildedge.logging import logger +from wildedge.model import ModelHandle +from wildedge.settings import parse_hub_list, parse_integration_list +from wildedge.tracing import trace_context + +_lock = threading.Lock() +_default_client: WildEdge | None = None + + +def set_default_client(client: WildEdge | None) -> None: + """Register ``client`` as the process default; ``None`` clears it.""" + global _default_client + with _lock: + _default_client = client + + +def peek_default_client() -> WildEdge | None: + """Return the process default client without creating one.""" + return _default_client + + +def get_client() -> WildEdge: + """ + Return the process default client, creating one from the environment on + first use. + + The lazily created client reads ``WILDEDGE_DSN`` exactly like a directly + constructed ``WildEdge`` (without a DSN it is a no-op) and enables only + the integrations and hubs named in ``WILDEDGE_INTEGRATIONS`` and + ``WILDEDGE_HUBS``. When those variables are unset nothing is patched: + auto-instrumentation stays opt-in. + """ + global _default_client + with _lock: + if _default_client is None: + _default_client = _client_from_env() + return _default_client + + +def _client_from_env() -> WildEdge: + client = WildEdge() + raw_integrations = os.environ.get(constants.ENV_INTEGRATIONS) + raw_hubs = os.environ.get(constants.ENV_HUBS) + integrations = ( + parse_integration_list(raw_integrations, sorted(supported_integrations())) + if raw_integrations + else [] + ) + hubs = parse_hub_list(raw_hubs, sorted(supported_hubs())) if raw_hubs else [] + + targets: list[tuple[str | None, list[str] | None]] = [] + if integrations: + targets = [(integration, hubs or None) for integration in integrations] + elif hubs: + targets = [(None, hubs)] + # A bad environment value must not break the caller that triggered lazy + # creation, so failures downgrade to a warning here; explicit init() with + # the same names would raise. + for integration, target_hubs in targets: + try: + client.instrument(integration, hubs=target_hubs) + except Exception as exc: + logger.warning( + "wildedge: instrument(%r) from environment failed: %s", + integration, + exc, + ) + return client + + +# A trace sets correlation contextvars and emits nothing, so it needs no +# client; the module-level name is the context manager itself. +trace = trace_context + + +def span( + *, + kind: SpanKind, + name: str, + status: SpanStatus = "ok", + model_id: str | None = None, + input_summary: str | None = None, + output_summary: str | None = None, + attributes: dict[str, Any] | None = None, + trace_id: str | None = None, + span_id: str | None = None, + parent_span_id: str | None = None, + run_id: str | None = None, + agent_id: str | None = None, + step_index: int | None = None, + conversation_id: str | None = None, + context: dict[str, Any] | None = None, +) -> SpanContextManager: + """``WildEdge.span`` on the default client.""" + return get_client().span( + kind=kind, + name=name, + status=status, + model_id=model_id, + input_summary=input_summary, + output_summary=output_summary, + attributes=attributes, + trace_id=trace_id, + span_id=span_id, + parent_span_id=parent_span_id, + run_id=run_id, + agent_id=agent_id, + step_index=step_index, + conversation_id=conversation_id, + context=context, + ) + + +def track_span( + *, + kind: SpanKind, + name: str, + duration_ms: int, + status: SpanStatus = "ok", + model_id: str | None = None, + input_summary: str | None = None, + output_summary: str | None = None, + attributes: dict[str, Any] | None = None, + trace_id: str | None = None, + span_id: str | None = None, + parent_span_id: str | None = None, + run_id: str | None = None, + agent_id: str | None = None, + step_index: int | None = None, + conversation_id: str | None = None, + context: dict[str, Any] | None = None, +) -> str: + """``WildEdge.track_span`` on the default client.""" + return get_client().track_span( + kind=kind, + name=name, + duration_ms=duration_ms, + status=status, + model_id=model_id, + input_summary=input_summary, + output_summary=output_summary, + attributes=attributes, + trace_id=trace_id, + span_id=span_id, + parent_span_id=parent_span_id, + run_id=run_id, + agent_id=agent_id, + step_index=step_index, + conversation_id=conversation_id, + context=context, + ) + + +def register_model( + model_obj: object, + *, + model_id: str | None = None, + source: str | None = None, + family: str | None = None, + version: str | None = None, + quantization: str | None = None, + auto_instrument: bool = True, +) -> ModelHandle: + """``WildEdge.register_model`` on the default client.""" + return get_client().register_model( + model_obj, + model_id=model_id, + source=source, + family=family, + version=version, + quantization=quantization, + auto_instrument=auto_instrument, + ) + + +def flush(timeout: float = 5.0) -> None: + """Flush the default client's queued events.""" + get_client().flush(timeout=timeout) diff --git a/wildedge/integrations/common.py b/wildedge/integrations/common.py index 1506b01..13be347 100644 --- a/wildedge/integrations/common.py +++ b/wildedge/integrations/common.py @@ -4,8 +4,10 @@ from collections.abc import Callable from typing import TYPE_CHECKING, Any +from urllib.parse import urlparse from wildedge import constants +from wildedge.events.inference import TextInputMeta from wildedge.logging import logger from wildedge.timing import elapsed_ms @@ -17,6 +19,73 @@ def debug_failure(framework: str, context: str, exc: BaseException) -> None: logger.debug("wildedge: %s %s failed: %s", framework, context, exc) +# --------------------------------------------------------------------------- +# Chat API helpers, shared by the openai integration and wildedge.llm_api +# --------------------------------------------------------------------------- + +SOURCE_BY_HOSTNAME: dict[str, str] = { + "api.openai.com": "openai", + "openrouter.ai": "openrouter", + "api.anthropic.com": "anthropic", + "api.mistral.ai": "mistral", + "api.groq.com": "groq", + "api.together.xyz": "together", + "api.deepseek.com": "deepseek", + "api.x.ai": "xai", + "generativelanguage.googleapis.com": "google", + "api.fireworks.ai": "fireworks", + "api.cerebras.ai": "cerebras", + "api.perplexity.ai": "perplexity", + "integrate.api.nvidia.com": "nvidia", + "router.huggingface.co": "huggingface", + "inference.baseten.co": "baseten", +} + +# Providers whose endpoints live on per-resource subdomains. +SOURCE_BY_HOSTNAME_SUFFIX: list[tuple[str, str]] = [ + (".openai.azure.com", "azure-openai"), + (".api.baseten.co", "baseten"), +] + + +def source_from_base_url(base_url: str | None) -> str: + hostname = urlparse(base_url.lower()).hostname if base_url else "" + if not hostname: + return "openai" + exact = SOURCE_BY_HOSTNAME.get(hostname) + if exact is not None: + return exact + for suffix, source in SOURCE_BY_HOSTNAME_SUFFIX: + if hostname.endswith(suffix): + return source + return hostname + + +def _msg_role(m) -> str | None: + return m.get("role") if isinstance(m, dict) else getattr(m, "role", None) + + +def _msg_content(m) -> str | None: + return m.get("content") if isinstance(m, dict) else getattr(m, "content", None) + + +def build_input_meta(messages: list, tokens_in: int | None) -> TextInputMeta | None: + if not messages: + return None + last_user = next((m for m in reversed(messages) if _msg_role(m) == "user"), None) + if not last_user: + return None + content = _msg_content(last_user) or "" + if not isinstance(content, str) or not content: + return None + return TextInputMeta( + char_count=len(content), + word_count=len(content.split()), + token_count=tokens_in, + prompt_type="chat", + ) + + DTYPE_QUANTIZATION_MAP: dict[str, str] = { "bfloat16": "bf16", "float16": "f16", diff --git a/wildedge/integrations/openai.py b/wildedge/integrations/openai.py index 700aace..b60de6c 100644 --- a/wildedge/integrations/openai.py +++ b/wildedge/integrations/openai.py @@ -6,15 +6,17 @@ import threading import time from typing import TYPE_CHECKING -from urllib.parse import urlparse from wildedge import constants -from wildedge.events.inference import ApiMeta, GenerationOutputMeta, TextInputMeta +from wildedge.events.inference import ApiMeta, GenerationOutputMeta from wildedge.integrations.base import BaseExtractor from wildedge.integrations.common import ( + SOURCE_BY_HOSTNAME, # noqa: F401 (re-exported; moved to common) AsyncStreamWrapper, SyncStreamWrapper, + build_input_meta, debug_failure, + source_from_base_url, ) from wildedge.model import ModelInfo from wildedge.timing import elapsed_ms @@ -34,42 +36,6 @@ debug_openai_failure = functools.partial(debug_failure, "openai") -SOURCE_BY_HOSTNAME: dict[str, str] = { - "api.openai.com": "openai", - "openrouter.ai": "openrouter", -} - - -def source_from_base_url(base_url: str | None) -> str: - hostname = urlparse(base_url.lower()).hostname if base_url else "" - return SOURCE_BY_HOSTNAME.get(hostname or "", hostname or "openai") - - -def _msg_role(m) -> str | None: - return m.get("role") if isinstance(m, dict) else getattr(m, "role", None) - - -def _msg_content(m) -> str | None: - return m.get("content") if isinstance(m, dict) else getattr(m, "content", None) - - -def build_input_meta(messages: list, tokens_in: int | None) -> TextInputMeta | None: - if not messages: - return None - last_user = next((m for m in reversed(messages) if _msg_role(m) == "user"), None) - if not last_user: - return None - content = _msg_content(last_user) or "" - if not isinstance(content, str) or not content: - return None - return TextInputMeta( - char_count=len(content), - word_count=len(content.split()), - token_count=tokens_in, - prompt_type="chat", - ) - - def build_streaming_output_meta( ttft_ms: int | None, tokens_in: int | None, diff --git a/wildedge/llm_api.py b/wildedge/llm_api.py new file mode 100644 index 0000000..8bb6236 --- /dev/null +++ b/wildedge/llm_api.py @@ -0,0 +1,260 @@ +"""Tracking for LLM calls made over an HTTP API. + +For applications that call an LLM API with a plain HTTP client (httpx or +requests against OpenRouter, vLLM, Ollama, or any OpenAI-compatible endpoint) +instead of the openai/anthropic client libraries that auto-instrumentation +patches. For models running inside your own process (llama.cpp, transformers, +MLX), use the framework integrations instead; they capture model metadata +this API boundary cannot see. Times the call, normalizes usage payloads from +either provider shape, and emits the same inference events as the +integrations: + + with wildedge.llm_api(model="openai/gpt-4o-mini", provider="openrouter") as call: + data = post_chat_completion(...) + call.response(data) + +Correlates with surrounding ``wildedge.trace`` / ``wildedge.span`` blocks +like any other event, and is a silent no-op without a DSN. +""" + +from __future__ import annotations + +import time +from typing import TYPE_CHECKING, Any + +from wildedge import constants +from wildedge.events.inference import ApiMeta, GenerationOutputMeta, TextInputMeta +from wildedge.integrations.common import build_input_meta, source_from_base_url +from wildedge.logging import logger +from wildedge.timing import elapsed_ms + +if TYPE_CHECKING: + from wildedge.model import ModelHandle + + +def _get(obj: object, name: str) -> Any: + if obj is None: + return None + if isinstance(obj, dict): + return obj.get(name) + return getattr(obj, name, None) + + +def _first(obj: object, *names: str) -> Any: + for name in names: + value = _get(obj, name) + if value is not None: + return value + return None + + +class LLMCall: + """Mutable record of one LLM API call; emits an inference event on exit. + + Fields may be set directly (``call.stop_reason = ...``, ``call.success = + False``) or through :meth:`usage` / :meth:`response` before the block + exits. An exception escaping the block records an error event instead, + with the exception class as the error code. + """ + + def __init__( + self, + *, + model: str, + provider: str | None = None, + base_url: str | None = None, + prompt: str | None = None, + messages: list | None = None, + ): + self.model = model + self.source = provider or ( + source_from_base_url(base_url) if base_url else "api" + ) + self.prompt = prompt + self.messages = messages + self.success = True + self.stop_reason: str | None = None + self.tokens_in: int | None = None + self.tokens_out: int | None = None + self.cached_tokens: int | None = None + self.reasoning_tokens: int | None = None + self.resolved_model_id: str | None = None + self.system_fingerprint: str | None = None + self.service_tier: str | None = None + self._t0: float | None = None + self._ttft_ms: int | None = None + + def first_token(self) -> None: + """Mark time-to-first-token for streaming responses.""" + if self._t0 is not None and self._ttft_ms is None: + self._ttft_ms = elapsed_ms(self._t0) + + def usage(self, payload: object = None, **fields: int | None) -> LLMCall: + """Record token usage from a raw usage payload or explicit fields. + + ``payload`` may be a dict or attribute object in OpenAI shape + (``prompt_tokens``/``completion_tokens`` with nested details) or + Anthropic shape (``input_tokens``/``output_tokens``, + ``cache_read_input_tokens``). Explicit keyword fields (``tokens_in``, + ``tokens_out``, ``cached_tokens``, ``reasoning_tokens``) win over the + payload. + """ + if payload is not None: + self.tokens_in = _first(payload, "prompt_tokens", "input_tokens") + self.tokens_out = _first(payload, "completion_tokens", "output_tokens") + cached = _get(_get(payload, "prompt_tokens_details"), "cached_tokens") + if cached is None: + cached = _get(payload, "cache_read_input_tokens") + self.cached_tokens = cached + self.reasoning_tokens = _get( + _get(payload, "completion_tokens_details"), "reasoning_tokens" + ) + for name in ("tokens_in", "tokens_out", "cached_tokens", "reasoning_tokens"): + if fields.get(name) is not None: + setattr(self, name, fields[name]) + return self + + def response(self, payload: object) -> LLMCall: + """Record usage, stop reason and API metadata from a full response. + + Accepts a chat-completion response as a dict or SDK object, in OpenAI + shape (``usage``, ``choices[0].finish_reason``, ``model``) or + Anthropic shape (``usage``, top-level ``stop_reason``). + """ + usage = _get(payload, "usage") + if usage is not None: + self.usage(usage) + choices = _get(payload, "choices") or [] + stop_reason = _get(choices[0], "finish_reason") if choices else None + if stop_reason is None: + stop_reason = _get(payload, "stop_reason") + if stop_reason is not None: + self.stop_reason = stop_reason + for field, name in ( + ("resolved_model_id", "model"), + ("system_fingerprint", "system_fingerprint"), + ("service_tier", "service_tier"), + ): + value = _get(payload, name) + if value is not None: + setattr(self, field, value) + return self + + def __enter__(self) -> LLMCall: + self._t0 = time.perf_counter() + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> bool: + if self._t0 is None: + return False + duration_ms = elapsed_ms(self._t0) + handle = self._handle() + if handle is None: + return False + if exc_type is not None: + handle.track_error( + error_code=exc_type.__name__, + error_message=str(exc_val)[: constants.ERROR_MSG_MAX_LEN], + ) + return False + handle.track_inference( + duration_ms=duration_ms, + input_modality="text", + output_modality="generation", + success=self.success, + input_meta=self._input_meta(), + output_meta=self._output_meta(duration_ms), + api_meta=self._api_meta(), + ) + return False + + async def __aenter__(self) -> LLMCall: + return self.__enter__() + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> bool: + return self.__exit__(exc_type, exc_val, exc_tb) + + def _handle(self) -> ModelHandle | None: + from wildedge.defaults import get_client # noqa: PLC0415 (import cycle) + + try: + return get_client().register_model( + None, model_id=self.model, source=self.source, model_format="api" + ) + except Exception as exc: + logger.debug("wildedge: llm model registration failed: %s", exc) + return None + + def _input_meta(self) -> TextInputMeta | None: + if self.messages: + return build_input_meta(self.messages, self.tokens_in) + if self.prompt: + return TextInputMeta( + char_count=len(self.prompt), + word_count=len(self.prompt.split()), + token_count=self.tokens_in, + prompt_type="chat", + ) + return None + + def _output_meta(self, duration_ms: int) -> GenerationOutputMeta | None: + known = ( + self.tokens_in, + self.tokens_out, + self.cached_tokens, + self.reasoning_tokens, + self.stop_reason, + self._ttft_ms, + ) + if all(value is None for value in known): + return None + tps = ( + round(self.tokens_out / duration_ms * 1000, 1) + if duration_ms > 0 and self.tokens_out + else None + ) + return GenerationOutputMeta( + task="generation", + tokens_in=self.tokens_in, + tokens_out=self.tokens_out, + cached_input_tokens=self.cached_tokens, + reasoning_tokens_out=self.reasoning_tokens, + time_to_first_token_ms=self._ttft_ms, + tokens_per_second=tps, + stop_reason=self.stop_reason, + ) + + def _api_meta(self) -> ApiMeta | None: + if not any( + [self.resolved_model_id, self.system_fingerprint, self.service_tier] + ): + return None + return ApiMeta( + resolved_model_id=self.resolved_model_id, + system_fingerprint=self.system_fingerprint, + service_tier=self.service_tier, + ) + + +def llm_api( + *, + model: str, + provider: str | None = None, + base_url: str | None = None, + prompt: str | None = None, + messages: list | None = None, +) -> LLMCall: + """Track one LLM API call made with any HTTP client. + + ``model`` is the model id the endpoint expects. ``provider`` names the + event source directly; alternatively pass ``base_url`` to derive it. + ``prompt`` or ``messages`` (chat format) enable input metadata. Usable as + a sync or async context manager. + """ + return LLMCall( + model=model, + provider=provider, + base_url=base_url, + prompt=prompt, + messages=messages, + ) diff --git a/wildedge/runtime/bootstrap.py b/wildedge/runtime/bootstrap.py index fbc47d7..63e0ee4 100644 --- a/wildedge/runtime/bootstrap.py +++ b/wildedge/runtime/bootstrap.py @@ -14,6 +14,7 @@ from wildedge import constants from wildedge.client import WildEdge from wildedge.constants import ENV_DSN, WILDEDGE_AUTOLOAD +from wildedge.defaults import set_default_client from wildedge.hubs.registry import HUBS_BY_NAME, supported_hubs from wildedge.integrations.registry import INTEGRATIONS_BY_NAME, supported_integrations from wildedge.settings import read_runtime_env @@ -27,6 +28,11 @@ STATUS_ERROR_PATCH_FAILED = "ERROR_PATCH_FAILED" STRICT_FAILURE_STATUSES = {STATUS_SKIP_MISSING_DEP, STATUS_ERROR_PATCH_FAILED} +# Exit codes used when strict mode turns a bootstrap failure fatal. +EXIT_CONFIG_ERROR = 120 +EXIT_STRICT_INTEGRATION_ERROR = 121 +EXIT_BOOTSTRAP_INTERNAL_ERROR = 122 + def clear_runtime_env() -> None: """Remove run-scoped env vars so nested processes do not inherit runtime config.""" @@ -38,6 +44,7 @@ def clear_runtime_env() -> None: constants.ENV_INTEGRATIONS, constants.ENV_HUBS, constants.ENV_STRICT_INTEGRATIONS, + constants.ENV_STRICT, constants.ENV_PROPAGATE, constants.ENV_PRINT_STARTUP_REPORT, constants.ENV_SAMPLING_INTERVAL, @@ -115,6 +122,7 @@ def install_runtime(*, install_signal_handlers: bool = True) -> RuntimeContext: debug=env.debug, sampling_interval_s=env.sampling_interval_s, ) + set_default_client(client) statuses: list[dict[str, str]] = [] for integration in env.integrations: diff --git a/wildedge/runtime/runner.py b/wildedge/runtime/runner.py deleted file mode 100644 index bda0009..0000000 --- a/wildedge/runtime/runner.py +++ /dev/null @@ -1,73 +0,0 @@ -"""Child process runner used by `wildedge run`.""" - -from __future__ import annotations - -import argparse -import runpy -import sys - -from wildedge.runtime.bootstrap import ( - RuntimeConfigError, - RuntimeStrictIntegrationError, - clear_runtime_env, - format_startup_report, - install_runtime, -) -from wildedge.settings import read_runner_env - -EXIT_CONFIG_ERROR = 120 -EXIT_STRICT_INTEGRATION_ERROR = 121 -EXIT_BOOTSTRAP_INTERNAL_ERROR = 122 - - -def _build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(prog="python -m wildedge.runtime.runner") - parser.add_argument("--mode", choices=["script", "module"], required=True) - parser.add_argument("--target", required=True) - parser.add_argument("args", nargs=argparse.REMAINDER) - return parser - - -def main(argv: list[str] | None = None) -> int: - parser = _build_parser() - parsed = parser.parse_args(argv) - args = parsed.args - if args and args[0] == "--": - args = args[1:] - - try: - context = install_runtime() - except RuntimeConfigError as exc: - print(f"wildedge: {exc}", file=sys.stderr) - return EXIT_CONFIG_ERROR - except RuntimeStrictIntegrationError as exc: - print(f"wildedge: {exc}", file=sys.stderr) - return EXIT_STRICT_INTEGRATION_ERROR - except Exception as exc: - print(f"wildedge: bootstrap internal error: {exc}", file=sys.stderr) - return EXIT_BOOTSTRAP_INTERNAL_ERROR - - runner_env = read_runner_env() - if ( - getattr(context, "debug", False) - or getattr(context, "print_startup_report", False) - or runner_env.print_startup_report - ): - print(format_startup_report(context), file=sys.stderr) - - if not runner_env.propagate: - clear_runtime_env() - try: - if parsed.mode == "script": - sys.argv = [parsed.target, *args] - runpy.run_path(parsed.target, run_name="__main__") - else: - sys.argv = [parsed.target, *args] - runpy.run_module(parsed.target, run_name="__main__", alter_sys=True) - finally: - context.shutdown() - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/wildedge/settings.py b/wildedge/settings.py index 138832d..6c445b8 100644 --- a/wildedge/settings.py +++ b/wildedge/settings.py @@ -35,6 +35,7 @@ class RuntimeEnv: class RunnerEnv: print_startup_report: bool propagate: bool + strict: bool def parse_bool(value: str | None) -> bool: @@ -134,4 +135,5 @@ def read_runner_env( return RunnerEnv( print_startup_report=parse_bool(env.get(constants.ENV_PRINT_STARTUP_REPORT)), propagate=parse_bool(env.get(constants.ENV_PROPAGATE, "1")), + strict=parse_bool(env.get(constants.ENV_STRICT)), )