From c2cd2ce21e871ddedefd465e8406ef88218bd6ec Mon Sep 17 00:00:00 2001 From: Piotr Duda Date: Tue, 14 Jul 2026 20:54:47 +0200 Subject: [PATCH 1/2] llms.txt + llm_api example + modernize examples to the module-level API --- .github/workflows/release.yml | 10 ++- CHANGELOG.md | 2 + README.md | 35 +++++++--- docs/llm_api.md | 3 + examples/agentic_example.py | 12 ++-- examples/anthropic_example.py | 4 +- examples/attachments_example.py | 6 +- examples/feedback_example.py | 4 +- examples/gguf_example.py | 2 +- examples/llm_api_example.py | 63 ++++++++++++++++++ examples/mlx_example.py | 4 +- examples/onnx_example.py | 2 +- examples/openai_example.py | 4 +- examples/tensorflow_example.py | 4 +- examples/timm_example.py | 2 +- examples/transformers_example.py | 4 +- scripts/build_llms_txt.py | 110 +++++++++++++++++++++++++++++++ tests/test_build_llms_txt.py | 57 ++++++++++++++++ 18 files changed, 293 insertions(+), 35 deletions(-) create mode 100644 examples/llm_api_example.py create mode 100644 scripts/build_llms_txt.py create mode 100644 tests/test_build_llms_txt.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ced414c..8eda977 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -47,11 +47,19 @@ jobs: OUTPUT: /tmp/release-notes.md run: python3 scripts/build_changelog_comment.py + - name: Build llms.txt artifacts + if: github.ref_type == 'tag' + env: + OUTPUT_DIR: llms-dist + run: python3 scripts/build_llms_txt.py + - name: Create GitHub release if: github.ref_type == 'tag' uses: softprops/action-gh-release@v2 with: - files: dist/* + files: | + dist/* + llms-dist/* body_path: /tmp/release-notes.md - name: Publish package to PyPI diff --git a/CHANGELOG.md b/CHANGELOG.md index 500e0fd..cb3c683 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ under Unreleased and move into a version section at release time. ### Added +- `examples/llm_api_example.py`: raw-HTTP LLM tracking with `wildedge.llm_api()`; existing examples updated to the module-level API (`wildedge.span` / `wildedge.flush` / `wildedge.register_model` instead of threading a client variable) +- Releases ship `llms.txt` and `llms-full.txt` as GitHub release assets: the full documentation for that exact version in one file, generated by `scripts/build_llms_txt.py`. README quickstart rewritten around the module-level API. - `wildedge doctor --send-test-event`: sends one real span event through the full pipeline and reports the ingest response, proving DSN auth and connectivity end to end. The report gains an `environment` section (`WILDEDGE_*` variables, autoload PYTHONPATH status) plus `config_status` / `connectivity_status` fields. - `wildedge.llm_api()`: provider-agnostic tracking for LLM calls made with any HTTP client (OpenRouter, vLLM, Ollama, OpenAI/Anthropic-compatible endpoints). Times the block, normalizes usage payloads from either provider shape via `call.usage()` / `call.response()`, supports TTFT marks and async use, and records exceptions as error events. See `docs/llm_api.md`. - `register_model()` without a matching extractor defaults the model name to the explicit `model_id` instead of the placeholder object's type name. diff --git a/README.md b/README.md index 31379b6..df60fd4 100644 --- a/README.md +++ b/README.md @@ -55,18 +55,23 @@ Useful flags: ```python import wildedge -client = wildedge.init( - dsn="...", # or WILDEDGE_DSN env var - integrations=["transformers"], - hubs=["huggingface"], -) +wildedge.init(integrations=["transformers"]) # optional under `wildedge run` -# models loaded after this point are tracked automatically -``` +# models loaded after this point are tracked automatically; add traces, +# spans and LLM API calls anywhere, no client instance to pass around: +with wildedge.trace(run_id="run-1"): + with wildedge.span(kind="agent_step", name="plan"): + ... -If no DSN is configured, the client becomes a no-op and logs a warning. +with wildedge.llm_api(model="openai/gpt-4o-mini", provider="openrouter") as call: + call.response(data) # LLM calls made with plain HTTP clients +``` -`init(...)` is a convenience wrapper for `WildEdge(...)` + `instrument(...)`. +One client per process: `wildedge run`, `init()`, and the module-level calls +all share it, and `init()` without `dsn` reuses whatever already exists. +Without a DSN everything is a silent no-op, so dev and CI need no +configuration. See [Deployment](https://github.com/wild-edge/wildedge-python/blob/main/docs/deployment.md) +for the full contract. ## Supported integrations **On-device** @@ -90,6 +95,10 @@ If no DSN is configured, the client becomes a no-op and logs a warning. | `anthropic` | [anthropic_example.py](https://github.com/wild-edge/wildedge-python/blob/main/examples/anthropic_example.py) | | `openai` | [openai_example.py](https://github.com/wild-edge/wildedge-python/blob/main/examples/openai_example.py) | +Calling an LLM API with a plain HTTP client instead of these libraries? Use +[`wildedge.llm_api()`](https://github.com/wild-edge/wildedge-python/blob/main/docs/llm_api.md): +[llm_api_example.py](https://github.com/wild-edge/wildedge-python/blob/main/examples/llm_api_example.py). + **Hub tracking** Pass `hubs=` to track model download provenance. Hubs are framework-agnostic and can be combined with any integration. @@ -135,6 +144,12 @@ Report security and privacy issues to: support@wildedge.dev ## Links +- [Deployment guide](https://github.com/wild-edge/wildedge-python/blob/main/docs/deployment.md) +- [Manual tracking](https://github.com/wild-edge/wildedge-python/blob/main/docs/manual-tracking.md) +- [LLM API tracking](https://github.com/wild-edge/wildedge-python/blob/main/docs/llm_api.md) - [Compatibility Matrix](https://github.com/wild-edge/wildedge-python/blob/main/docs/compatibility.md) -- [Changelog](https://github.com/wild-edge/wildedge-python/releases) +- [Changelog](https://github.com/wild-edge/wildedge-python/blob/main/CHANGELOG.md) - [License](https://github.com/wild-edge/wildedge-python/blob/main/LICENSE) + +Each GitHub release ships `llms.txt` and `llms-full.txt`: the full +documentation for that exact version in one file, built for AI assistants. diff --git a/docs/llm_api.md b/docs/llm_api.md index 0fc1525..7248a2c 100644 --- a/docs/llm_api.md +++ b/docs/llm_api.md @@ -41,6 +41,9 @@ full response payload, dict or SDK object, OpenAI shape (`usage.prompt_tokens`, `choices[0].finish_reason`) or Anthropic shape (`usage.input_tokens`, top-level `stop_reason`). +Runnable version: [examples/llm_api_example.py](../examples/llm_api_example.py), +stdlib urllib against OpenRouter, no client library at all. + ## Recording pieces individually When you do not have a full response payload, set what you know: diff --git a/examples/agentic_example.py b/examples/agentic_example.py index 446c812..6874184 100644 --- a/examples/agentic_example.py +++ b/examples/agentic_example.py @@ -26,7 +26,7 @@ import wildedge -we = wildedge.init( +wildedge.init( app_version="1.0.0", integrations="openai", ) @@ -96,7 +96,7 @@ def calculator(expression: str) -> str: def call_tool(name: str, arguments: dict) -> str: - with we.span( + with wildedge.span( kind="tool", name=name, input_summary=json.dumps(arguments)[:200], @@ -108,7 +108,7 @@ def call_tool(name: str, arguments: dict) -> str: def retrieve_context(query: str) -> str: """Fetch relevant context from the vector store (~120ms).""" - with we.span( + with wildedge.span( kind="retrieval", name="vector_search", input_summary=query[:200], @@ -125,7 +125,7 @@ def run_agent(task: str, step_index: int, messages: list) -> str: messages.append({"role": "user", "content": f"{task}\n\nContext: {context}"}) while True: - with we.span( + with wildedge.span( kind="agent_step", name="reason", step_index=step_index, @@ -172,10 +172,10 @@ def run_agent(task: str, step_index: int, messages: list) -> str: system_prompt = "You are a helpful assistant. Use tools when needed." messages = [{"role": "system", "content": system_prompt}] -with we.trace(agent_id="demo-agent", run_id=str(uuid.uuid4())): +with wildedge.trace(agent_id="demo-agent", run_id=str(uuid.uuid4())): for i, task in enumerate(TASKS, start=1): print(f"\nTask {i}: {task}") reply = run_agent(task, step_index=i, messages=messages) print(f"Reply: {reply}") -we.flush() +wildedge.flush() diff --git a/examples/anthropic_example.py b/examples/anthropic_example.py index 62fca35..eab152c 100644 --- a/examples/anthropic_example.py +++ b/examples/anthropic_example.py @@ -18,7 +18,7 @@ import wildedge -client = wildedge.init( +wildedge.init( app_version="1.0.0", # uses WILDEDGE_DSN if set; otherwise no-op integrations="anthropic", ) @@ -44,5 +44,5 @@ print(event.delta.text, end="", flush=True) print("\n") -client.flush() +wildedge.flush() print("Done. Events flushed to WildEdge.") diff --git a/examples/attachments_example.py b/examples/attachments_example.py index 4eb0735..4669561 100644 --- a/examples/attachments_example.py +++ b/examples/attachments_example.py @@ -26,7 +26,7 @@ def redact(attachments: list[Attachment]) -> list[Attachment]: return [a for a in attachments if a.content_type != "application/secret"] -client = wildedge.init( +wildedge.init( app_version="1.0.0", attachments_enabled=True, max_attachments_per_inference=5, @@ -35,7 +35,7 @@ def redact(attachments: list[Attachment]) -> list[Attachment]: attachment_filter=redact, ) -handle = client.register_model( +handle = wildedge.register_model( object(), model_id="doc-classifier-v1", source="local", @@ -59,4 +59,4 @@ def redact(attachments: list[Attachment]) -> list[Attachment]: print(f"tracked inference {inference_id[:8]}… with 2 attachments") # Bytes upload in the background; flush/close lets buffered events drain. -client.close() +wildedge.flush() diff --git a/examples/feedback_example.py b/examples/feedback_example.py index ff06292..a1896dd 100644 --- a/examples/feedback_example.py +++ b/examples/feedback_example.py @@ -21,14 +21,14 @@ CONFIDENCE_THRESHOLD = 0.6 -client = wildedge.init( +wildedge.init( app_version="1.0.0", # uses WILDEDGE_DSN if set; otherwise no-op integrations="timm", ) model = timm.create_model("resnet18", pretrained=True) model.eval() -handle = client.register_model( +handle = wildedge.register_model( model ) # auto-instrumented already; returns existing handle diff --git a/examples/gguf_example.py b/examples/gguf_example.py index bd50f98..861603d 100644 --- a/examples/gguf_example.py +++ b/examples/gguf_example.py @@ -12,7 +12,7 @@ import wildedge -client = wildedge.init( +wildedge.init( app_version="1.0.0", # uses WILDEDGE_DSN if set; otherwise no-op integrations="gguf", hubs=["huggingface"], diff --git a/examples/llm_api_example.py b/examples/llm_api_example.py new file mode 100644 index 0000000..1afe2ff --- /dev/null +++ b/examples/llm_api_example.py @@ -0,0 +1,63 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = ["wildedge-sdk"] +# +# [tool.uv.sources] +# wildedge-sdk = { path = "..", editable = true } +# /// +"""Track LLM calls made over plain HTTP with wildedge.llm_api(). + +No openai or anthropic client library involved: requests go through stdlib +urllib against OpenRouter's OpenAI-compatible endpoint, and llm_api() records +the same inference events the integrations emit (tokens, TTFT, stop reason), +correlated into the surrounding trace and spans. + +Run with: uv run llm_api_example.py +Requires: OPENROUTER_API_KEY environment variable. Set WILDEDGE_DSN to send events. +""" + +import json +import os +import urllib.request +import uuid + +import wildedge + +wildedge.init(app_version="1.0.0") # uses WILDEDGE_DSN if set; otherwise no-op + +URL = "https://openrouter.ai/api/v1/chat/completions" +MODEL = "openai/gpt-4o-mini" + + +def chat(prompt: str) -> dict: + request = urllib.request.Request( + URL, + data=json.dumps( + {"model": MODEL, "messages": [{"role": "user", "content": prompt}]} + ).encode(), + headers={ + "Authorization": f"Bearer {os.getenv('OPENROUTER_API_KEY')}", + "Content-Type": "application/json", + }, + ) + with urllib.request.urlopen(request, timeout=120) as response: + return json.load(response) + + +PROMPTS = [ + "What is on-device AI in one sentence?", + "Name three edge inference runtimes.", +] + +with wildedge.trace(agent_id="llm-api-example", run_id=str(uuid.uuid4())): + for step, prompt in enumerate(PROMPTS): + with wildedge.span(kind="agent_step", name="ask", step_index=step): + with wildedge.llm_api( + model=MODEL, provider="openrouter", prompt=prompt + ) as call: + data = chat(prompt) + call.response(data) + print(f"Q: {prompt}\nA: {data['choices'][0]['message']['content']}\n") + +wildedge.flush() +print("Done. Events flushed to WildEdge.") diff --git a/examples/mlx_example.py b/examples/mlx_example.py index 61906e0..e7c0068 100644 --- a/examples/mlx_example.py +++ b/examples/mlx_example.py @@ -52,7 +52,7 @@ def main() -> None: # init() constructs the client and instruments mlx; must be called # before any model is loaded. - client = wildedge.init( + wildedge.init( app_version="1.0.0", # uses WILDEDGE_DSN if set; otherwise no-op integrations="mlx", hubs=["huggingface"], @@ -73,7 +73,7 @@ def main() -> None: print(f"[{i}] Q: {prompt}") print(f" A: {response}\n") - client.flush() + wildedge.flush() print("Done. Events flushed to WildEdge.") diff --git a/examples/onnx_example.py b/examples/onnx_example.py index e7a6f78..eb91f09 100644 --- a/examples/onnx_example.py +++ b/examples/onnx_example.py @@ -13,7 +13,7 @@ import wildedge -client = wildedge.init( +wildedge.init( app_version="1.0.0", # uses WILDEDGE_DSN if set; otherwise no-op integrations="onnx", hubs=["huggingface"], diff --git a/examples/openai_example.py b/examples/openai_example.py index ad2d3a9..5179b82 100644 --- a/examples/openai_example.py +++ b/examples/openai_example.py @@ -18,7 +18,7 @@ import wildedge -client = wildedge.init( +wildedge.init( app_version="1.0.0", # uses WILDEDGE_DSN if set; otherwise no-op integrations="openai", ) @@ -46,5 +46,5 @@ print(chunk.choices[0].delta.content, end="", flush=True) print("\n") -client.flush() +wildedge.flush() print("Done. Events flushed to WildEdge.") diff --git a/examples/tensorflow_example.py b/examples/tensorflow_example.py index 3cf1962..cca72a8 100644 --- a/examples/tensorflow_example.py +++ b/examples/tensorflow_example.py @@ -17,7 +17,7 @@ import wildedge -client = wildedge.init( +wildedge.init( app_version="1.0.0", # uses WILDEDGE_DSN if set; otherwise no-op integrations="tensorflow", ) @@ -48,4 +48,4 @@ def build_and_save_model(save_path: Path) -> None: print("output shape:", tuple(output.shape)) -client.close() +wildedge.flush() diff --git a/examples/timm_example.py b/examples/timm_example.py index 5b4ffa1..5930368 100644 --- a/examples/timm_example.py +++ b/examples/timm_example.py @@ -19,7 +19,7 @@ import wildedge -client = wildedge.init( +wildedge.init( app_version="1.0.0", # uses WILDEDGE_DSN if set; otherwise no-op integrations="timm", hubs=["huggingface", "torchhub"], diff --git a/examples/transformers_example.py b/examples/transformers_example.py index 92e8a1d..4c0e53a 100644 --- a/examples/transformers_example.py +++ b/examples/transformers_example.py @@ -92,7 +92,7 @@ def main() -> None: ) args = parser.parse_args() - client = wildedge.init( + wildedge.init( app_version="1.0.0", # uses WILDEDGE_DSN if set; otherwise no-op integrations="transformers", hubs=["huggingface"], @@ -103,7 +103,7 @@ def main() -> None: args.task ]() - client.flush() + wildedge.flush() print("\nDone. Events flushed to WildEdge.") diff --git a/scripts/build_llms_txt.py b/scripts/build_llms_txt.py new file mode 100644 index 0000000..1a47e69 --- /dev/null +++ b/scripts/build_llms_txt.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Build llms.txt (index) and llms-full.txt (full content) from README + docs. + +llms.txt is a link index per https://llmstxt.org; llms-full.txt is the entire +documentation concatenated and stamped with the release version, so an agent +gets correct, versioned docs in one fetch. Run at release time; outputs land +in OUTPUT_DIR (default: llms-dist). +""" + +from __future__ import annotations + +import os +from datetime import datetime, timezone +from pathlib import Path + +import tomllib + +REPO_ROOT = Path(__file__).parent.parent + +# Order is the reading order in llms-full.txt. +SOURCES: list[tuple[str, str, str]] = [ + ("README.md", "", "WildEdge Python SDK"), + ("docs/configuration.md", "configuration", "Configuration"), + ("docs/deployment.md", "deployment", "Deployment"), + ("docs/manual-tracking.md", "manual-tracking", "Manual tracking"), + ("docs/llm_api.md", "llm_api", "LLM API tracking"), + ("docs/compatibility.md", "compatibility", "Compatibility matrix"), +] + + +def project_version() -> str: + data = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text()) + return data["project"]["version"] + + +def clean_markdown(text: str) -> str: + """Drop HTML tags and badge lines; they are noise to a text consumer.""" + kept = [ + line for line in text.splitlines() if not line.lstrip().startswith(("<", "[![")) + ] + return "\n".join(kept).strip() + "\n" + + +def first_paragraph(text: str) -> str: + """First prose paragraph outside code fences, as the index description.""" + collected: list[str] = [] + in_fence = False + for line in clean_markdown(text).splitlines(): + stripped = line.strip() + if stripped.startswith("```"): + in_fence = not in_fence + continue + if in_fence: + continue + if stripped.startswith(("#", ">", "|", "-")): + if collected: + break + continue + if not stripped: + if collected: + break + continue + collected.append(stripped) + paragraph = " ".join(collected) + return paragraph if len(paragraph) <= 220 else paragraph[:217] + "..." + + +def build(output_dir: Path, base_url: str) -> None: + version = project_version() + stamp = ( + f"wildedge-sdk {version} documentation, generated " + f"{datetime.now(timezone.utc).date().isoformat()} from the v{version} release." + ) + + index_lines = [ + "# WildEdge Python SDK", + "", + f"> {stamp}", + "", + "On-device ML inference monitoring for Python.", + "", + "## Docs", + "", + ] + full_parts = [f"# WildEdge Python SDK ({version})\n\n> {stamp}\n"] + + for relpath, slug, title in SOURCES: + path = REPO_ROOT / relpath + text = path.read_text(encoding="utf-8") + url = f"{base_url}/{slug}" if slug else base_url + description = first_paragraph(text) + index_lines.append(f"- [{title}]({url}): {description}") + full_parts.append(f"\n---\n\n{clean_markdown(text)}") + + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "llms.txt").write_text( + "\n".join(index_lines) + "\n", encoding="utf-8" + ) + (output_dir / "llms-full.txt").write_text("".join(full_parts), encoding="utf-8") + print(f"wrote {output_dir / 'llms.txt'} and {output_dir / 'llms-full.txt'}") + + +def main() -> None: + output_dir = Path(os.environ.get("OUTPUT_DIR", "llms-dist")) + base_url = os.environ.get("BASE_URL", "https://docs.wildedge.dev").rstrip("/") + build(output_dir, base_url) + + +if __name__ == "__main__": + main() diff --git a/tests/test_build_llms_txt.py b/tests/test_build_llms_txt.py new file mode 100644 index 0000000..23c9800 --- /dev/null +++ b/tests/test_build_llms_txt.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + +SCRIPT = Path(__file__).parent.parent / "scripts" / "build_llms_txt.py" + + +def load_script(): + spec = importlib.util.spec_from_file_location("build_llms_txt", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_build_produces_index_and_full_text(tmp_path): + module = load_script() + module.build(tmp_path, "https://docs.wildedge.dev") + + index = (tmp_path / "llms.txt").read_text() + full = (tmp_path / "llms-full.txt").read_text() + version = module.project_version() + + assert index.startswith("# WildEdge Python SDK") + assert f"wildedge-sdk {version} documentation" in index + assert "- [WildEdge Python SDK](https://docs.wildedge.dev):" in index + for slug in ("configuration", "deployment", "manual-tracking", "llm_api"): + assert f"https://docs.wildedge.dev/{slug}" in index + + assert f"# WildEdge Python SDK ({version})" in full + # One recognizable phrase per source document proves each was included. + assert "Full reference for all `WildEdge` client parameters" in full + assert "pre-deploy" in full or "wildedge run" in full + assert "register_model" in full + assert "llm_api" in full + + +def test_clean_markdown_strips_html_and_badges(): + module = load_script() + text = '

logo

\n[![CI](x)](y)\n# Title\nprose line\n' + cleaned = module.clean_markdown(text) + assert " Date: Tue, 14 Jul 2026 21:02:45 +0200 Subject: [PATCH 2/2] dependency fix --- scripts/build_llms_txt.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/build_llms_txt.py b/scripts/build_llms_txt.py index 1a47e69..5991ed6 100644 --- a/scripts/build_llms_txt.py +++ b/scripts/build_llms_txt.py @@ -10,11 +10,10 @@ from __future__ import annotations import os +import re from datetime import datetime, timezone from pathlib import Path -import tomllib - REPO_ROOT = Path(__file__).parent.parent # Order is the reading order in llms-full.txt. @@ -29,8 +28,13 @@ def project_version() -> str: - data = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text()) - return data["project"]["version"] + # Regex instead of tomllib: the script must run on 3.10, where tomllib + # does not exist, and this is the only field it needs. + text = (REPO_ROOT / "pyproject.toml").read_text() + match = re.search(r'^version = "([^"]+)"', text, re.MULTILINE) + if match is None: + raise SystemExit("could not find project version in pyproject.toml") + return match.group(1) def clean_markdown(text: str) -> str: