diff --git a/CHANGELOG.md b/CHANGELOG.md index cb3c683..cd0c34e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ under Unreleased and move into a version section at release time. ### Added +- `register_model()` accepts a `model_format` override for models without a matching extractor; `llm_api()` registers its models as format `"api"`, matching the openai/anthropic integrations instead of `"unknown"`. +- `source_from_base_url()` recognizes more provider hosts (anthropic, mistral, groq, together, deepseek, xai, google, fireworks, cerebras, perplexity, nvidia, huggingface, baseten) instead of falling back to raw hostnames, including suffix matching for per-resource subdomains (Azure OpenAI, Baseten). - `examples/llm_api_example.py`: raw-HTTP LLM tracking with `wildedge.llm_api()`; existing examples updated to the module-level API (`wildedge.span` / `wildedge.flush` / `wildedge.register_model` instead of threading a client variable) - Releases ship `llms.txt` and `llms-full.txt` as GitHub release assets: the full documentation for that exact version in one file, generated by `scripts/build_llms_txt.py`. README quickstart rewritten around the module-level API. - `wildedge doctor --send-test-event`: sends one real span event through the full pipeline and reports the ingest response, proving DSN auth and connectivity end to end. The report gains an `environment` section (`WILDEDGE_*` variables, autoload PYTHONPATH status) plus `config_status` / `connectivity_status` fields. diff --git a/docs/manual-tracking.md b/docs/manual-tracking.md index c53763e..ed52a24 100644 --- a/docs/manual-tracking.md +++ b/docs/manual-tracking.md @@ -81,11 +81,12 @@ For remote APIs with no local object to inspect, pass a placeholder: ```python handle = client.register_model( - object(), + None, model_id="openai/gpt-4o", - source="https://api.openai.com", + source="openai", family="gpt-4o", version="2024-08-06", + model_format="api", ) ``` diff --git a/scripts/check_tag_version.py b/scripts/check_tag_version.py index 08169f5..b20ff7a 100644 --- a/scripts/check_tag_version.py +++ b/scripts/check_tag_version.py @@ -4,10 +4,9 @@ from __future__ import annotations import os +import re from pathlib import Path -import tomllib - def main() -> None: tag = os.environ["TAG_NAME"] @@ -15,8 +14,13 @@ def main() -> None: raise SystemExit(f"Expected tag to start with 'v', got: {tag}") tag_version = tag[1:] - data = tomllib.loads(Path("pyproject.toml").read_text()) - project_version = data["project"]["version"] + # Regex instead of tomllib so the script also runs on Python 3.10. + match = re.search( + r'^version = "([^"]+)"', Path("pyproject.toml").read_text(), re.MULTILINE + ) + if match is None: + raise SystemExit("could not find project version in pyproject.toml") + project_version = match.group(1) if tag_version != project_version: raise SystemExit( diff --git a/tests/test_client.py b/tests/test_client.py index 56a2a2b..281f4db 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -201,3 +201,31 @@ def test_app_identity_env_override_used_for_paths(monkeypatch): p.assert_called_once_with("env-app") d.assert_called_once_with("env-app") r.assert_called_once_with("env-app") + + +def test_register_model_defaults_name_to_model_id_and_accepts_format(): + from wildedge.client import WildEdge + + client = WildEdge(dsn="https://secret@ingest.wildedge.dev/key") + client.registry.register.return_value = (object(), True) + + client.register_model( + None, model_id="org/model", source="openrouter", model_format="api" + ) + + info = client.registry.register.call_args[0][1] + assert info.model_name == "org/model" + assert info.model_source == "openrouter" + assert info.model_format == "api" + + +def test_register_model_format_defaults_to_unknown(): + from wildedge.client import WildEdge + + client = WildEdge(dsn="https://secret@ingest.wildedge.dev/key") + client.registry.register.return_value = (object(), True) + + client.register_model(None, model_id="org/model") + + info = client.registry.register.call_args[0][1] + assert info.model_format == "unknown" diff --git a/tests/test_integrations_openai.py b/tests/test_integrations_openai.py index dbba8d0..d15c622 100644 --- a/tests/test_integrations_openai.py +++ b/tests/test_integrations_openai.py @@ -183,7 +183,19 @@ def register_model(obj, *, model_id=None, source=None, **kwargs): ("", "openai"), ("https://openrouter.ai/api/v1", "openrouter"), ("https://api.openai.com/v1", "openai"), - ("https://api.together.xyz/v1", "api.together.xyz"), + ("https://api.together.xyz/v1", "together"), + ("https://api.anthropic.com", "anthropic"), + ("https://api.mistral.ai/v1", "mistral"), + ("https://api.groq.com/openai/v1", "groq"), + ("https://generativelanguage.googleapis.com/v1beta/openai", "google"), + ("https://api.fireworks.ai/inference/v1", "fireworks"), + ("https://myresource.openai.azure.com/openai/v1", "azure-openai"), + ( + "https://model-abc123.api.baseten.co/environments/production/sync/v1", + "baseten", + ), + ("https://inference.baseten.co/v1", "baseten"), + ("https://unknown.example.com/v1", "unknown.example.com"), ("https://localhost:11434/v1", "localhost"), ], ) diff --git a/tests/test_llm_api.py b/tests/test_llm_api.py index 5f21f11..ba992a4 100644 --- a/tests/test_llm_api.py +++ b/tests/test_llm_api.py @@ -221,3 +221,10 @@ def test_noop_without_dsn_end_to_end(monkeypatch): call.usage(tokens_in=1, tokens_out=2) assert wildedge.get_client().noop is True + + +def test_registers_model_as_api_format(client): + with wildedge.llm_api(model="m", provider="p"): + pass + + assert client.registered[0]["model_format"] == "api" diff --git a/wildedge/client.py b/wildedge/client.py index 61a6051..8787664 100644 --- a/wildedge/client.py +++ b/wildedge/client.py @@ -502,13 +502,15 @@ def register_model( family: str | None = None, version: str | None = None, quantization: str | None = None, + model_format: str | None = None, auto_instrument: bool = True, ) -> ModelHandle: """ Register a model and return a handle for tracking events. Auto-extracts metadata from ONNX Runtime and GGUF/llama.cpp objects. - User-supplied kwargs override extracted values. + User-supplied kwargs override extracted values. ``model_format`` + applies when no extractor matches (e.g. ``"api"`` for remote models). """ overrides = { k: v @@ -518,6 +520,7 @@ def register_model( "family": family, "version": version, "quantization": quantization, + "format": model_format, }.items() if v is not None } diff --git a/wildedge/integrations/common.py b/wildedge/integrations/common.py index 9d35c54..13be347 100644 --- a/wildedge/integrations/common.py +++ b/wildedge/integrations/common.py @@ -26,12 +26,39 @@ def debug_failure(framework: str, context: str, exc: BaseException) -> None: SOURCE_BY_HOSTNAME: dict[str, str] = { "api.openai.com": "openai", "openrouter.ai": "openrouter", + "api.anthropic.com": "anthropic", + "api.mistral.ai": "mistral", + "api.groq.com": "groq", + "api.together.xyz": "together", + "api.deepseek.com": "deepseek", + "api.x.ai": "xai", + "generativelanguage.googleapis.com": "google", + "api.fireworks.ai": "fireworks", + "api.cerebras.ai": "cerebras", + "api.perplexity.ai": "perplexity", + "integrate.api.nvidia.com": "nvidia", + "router.huggingface.co": "huggingface", + "inference.baseten.co": "baseten", } +# Providers whose endpoints live on per-resource subdomains. +SOURCE_BY_HOSTNAME_SUFFIX: list[tuple[str, str]] = [ + (".openai.azure.com", "azure-openai"), + (".api.baseten.co", "baseten"), +] + def source_from_base_url(base_url: str | None) -> str: hostname = urlparse(base_url.lower()).hostname if base_url else "" - return SOURCE_BY_HOSTNAME.get(hostname or "", hostname or "openai") + if not hostname: + return "openai" + exact = SOURCE_BY_HOSTNAME.get(hostname) + if exact is not None: + return exact + for suffix, source in SOURCE_BY_HOSTNAME_SUFFIX: + if hostname.endswith(suffix): + return source + return hostname def _msg_role(m) -> str | None: diff --git a/wildedge/llm_api.py b/wildedge/llm_api.py index 5f36c70..8bb6236 100644 --- a/wildedge/llm_api.py +++ b/wildedge/llm_api.py @@ -179,7 +179,7 @@ def _handle(self) -> ModelHandle | None: try: return get_client().register_model( - None, model_id=self.model, source=self.source + None, model_id=self.model, source=self.source, model_format="api" ) except Exception as exc: logger.debug("wildedge: llm model registration failed: %s", exc)