diff --git a/README.md b/README.md index 4198e91..76a6de8 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,8 @@ For the Week 4 release gates, readiness verdict, and next-stage priorities, see For the phased commercialization roadmap with milestones and success metrics, see `docs/21-commercialization-roadmap.md`. +For the Week 5 provider telemetry contract and privacy boundary, see `docs/22-provider-telemetry.md`. + Ask the assistant directly: ```bash @@ -144,6 +146,7 @@ AIOps-Lab/ 19-advanced-model-serving-roadmap.md 20-production-readiness-review.md 21-commercialization-roadmap.md + 22-provider-telemetry.md infra/ # Docker, Kubernetes, and Terraform starter notes k8s/ # kind-first Kubernetes manifests and walkthrough scripts/ # Local traffic and log helper scripts @@ -174,6 +177,12 @@ You should see the assistant report intentionally generated 500s, latency spikes - Week 2: observability basics, metrics, dashboards, structured logging, incident examples. - Week 3: Kubernetes manifests, operations, config/secrets, probes/resources, incident debugging, production next steps. - Week 4: security hardening, cost optimization, evaluation, production observability, advanced serving decisions, and a release-gated production-readiness review. +- Week 5: privacy-safe provider identity, latency, usage, outcome, fallback, and cost telemetry. +- Week 6: a larger versioned evaluation corpus with machine-readable CI regression reports. +- Week 7: an optional OpenTelemetry signal path plus one owned alert-to-runbook exercise. +- Week 8: a provider-versus-private-endpoint benchmark and an evidence-backed build-versus-buy decision. + +The technical weeks run alongside the commercialization phases; they do not bypass audience, design-partner, or willingness-to-pay gates. See `docs/09-roadmap.md` and `docs/21-commercialization-roadmap.md`. ## Build In Public diff --git a/apps/ai-sre-assistant/README.md b/apps/ai-sre-assistant/README.md index 840e0ef..1738798 100644 --- a/apps/ai-sre-assistant/README.md +++ b/apps/ai-sre-assistant/README.md @@ -80,6 +80,14 @@ API responses that attempt LLM enrichment include `llm_cost_controls` so callers See `../../docs/16-cost-optimization.md` for the Day 3 cost optimization guide. +## Provider Telemetry + +When `use_llm=true`, LLM-enabled API responses include `llm_telemetry` with bounded provider and model labels, configuration and attempt state, success or fallback outcome, request latency, and normalized token usage when the provider reports it. + +The telemetry object never includes prompts, incident evidence, credentials, provider base URLs, generated output, or user and incident identifiers. It is per-request metadata, not yet a persistent usage ledger, billing record, cost estimate, or Prometheus metrics implementation. + +See `../../docs/22-provider-telemetry.md` for field semantics, privacy rules, failure behavior, and the remaining Week 5 sequence. + ## Evaluation The assistant includes a deterministic incident corpus and a five-part quality rubric. Run it locally from this directory: diff --git a/apps/ai-sre-assistant/app/llm.py b/apps/ai-sre-assistant/app/llm.py index 6b275b1..a7b6f22 100644 --- a/apps/ai-sre-assistant/app/llm.py +++ b/apps/ai-sre-assistant/app/llm.py @@ -1,5 +1,6 @@ import json import os +import time from dataclasses import dataclass from typing import Any @@ -12,6 +13,7 @@ DEFAULT_LLM_MAX_LOG_ENTRIES = 50 DEFAULT_LLM_MAX_PROMPT_CHARS = 12000 TRUNCATION_MARKER = "\n...[truncated by LLM cost controls]\n" +MAX_TELEMETRY_LABEL_CHARS = 100 @dataclass(frozen=True) @@ -24,6 +26,13 @@ class LLMConfig: max_prompt_chars: int = DEFAULT_LLM_MAX_PROMPT_CHARS +@dataclass(frozen=True) +class LLMCallResult: + analysis: str | None + notice: str | None + telemetry: dict[str, Any] + + def load_config() -> LLMConfig: provider = os.getenv("LLM_PROVIDER", "none").strip().lower() default_base_url = "http://localhost:11434/v1" if provider == "ollama" else "https://api.openai.com/v1" @@ -57,10 +66,21 @@ def analyze_with_llm( logs: list[dict[str, Any]], rule_based_analysis: dict[str, Any], config: LLMConfig | None = None, -) -> tuple[str | None, str | None]: +) -> LLMCallResult: config = config or load_config() if not is_configured(config): - return None, "LLM provider is not configured; using rule-based analysis." + return LLMCallResult( + analysis=None, + notice="LLM provider is not configured; using rule-based analysis.", + telemetry=_telemetry( + config=config, + configured=False, + attempted=False, + outcome="not_configured", + fallback_used=True, + fallback_reason="provider_not_configured", + ), + ) user_prompt, prompt_was_truncated = build_user_prompt( question=question, @@ -87,18 +107,47 @@ def analyze_with_llm( "temperature": 0.2, } + request_started = time.perf_counter() try: with httpx.Client(timeout=30) as client: response = client.post(f"{config.base_url}/chat/completions", headers=headers, json=payload) response.raise_for_status() data = response.json() content = data["choices"][0]["message"]["content"] - return redact_text(str(content).strip()), cost_notice + analysis = redact_text(str(content).strip()) + if not analysis: + raise ValueError("LLM response content was empty") + return LLMCallResult( + analysis=analysis, + notice=cost_notice, + telemetry=_telemetry( + config=config, + configured=True, + attempted=True, + outcome="success", + fallback_used=False, + fallback_reason=None, + request_latency_ms=_elapsed_ms(request_started), + usage=data.get("usage"), + ), + ) except Exception as exc: failure_notice = f"LLM request failed; using rule-based analysis. Error: {redact_text(str(exc))}" if cost_notice: failure_notice = f"{cost_notice} {failure_notice}" - return None, failure_notice + return LLMCallResult( + analysis=None, + notice=failure_notice, + telemetry=_telemetry( + config=config, + configured=True, + attempted=True, + outcome="failure", + fallback_used=True, + fallback_reason="provider_request_failed", + request_latency_ms=_elapsed_ms(request_started), + ), + ) def build_user_prompt( @@ -154,3 +203,57 @@ def _truncate_text(value: str, max_chars: int) -> tuple[str, bool]: return value[:max_chars], True return value[: max_chars - len(TRUNCATION_MARKER)].rstrip() + TRUNCATION_MARKER, True + +def _telemetry( + config: LLMConfig, + *, + configured: bool, + attempted: bool, + outcome: str, + fallback_used: bool, + fallback_reason: str | None, + request_latency_ms: float | None = None, + usage: Any = None, +) -> dict[str, Any]: + return { + "provider": _bounded_label(config.provider or "none"), + "model": _bounded_label(config.model) if configured else None, + "configured": configured, + "attempted": attempted, + "outcome": outcome, + "request_latency_ms": request_latency_ms, + "fallback_used": fallback_used, + "fallback_reason": fallback_reason, + "usage": _normalize_usage(usage), + } + + +def _normalize_usage(usage: Any) -> dict[str, int | bool | None]: + if not isinstance(usage, dict): + usage = {} + + input_tokens = _non_negative_int(usage.get("prompt_tokens")) + output_tokens = _non_negative_int(usage.get("completion_tokens")) + total_tokens = _non_negative_int(usage.get("total_tokens")) + if total_tokens is None and input_tokens is not None and output_tokens is not None: + total_tokens = input_tokens + output_tokens + + return { + "reported": any(value is not None for value in (input_tokens, output_tokens, total_tokens)), + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": total_tokens, + } + + +def _non_negative_int(value: Any) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None + + +def _bounded_label(value: str) -> str: + return redact_text(str(value).strip())[:MAX_TELEMETRY_LABEL_CHARS] + + +def _elapsed_ms(started: float) -> float: + return round(max(0.0, (time.perf_counter() - started) * 1000), 2) + diff --git a/apps/ai-sre-assistant/app/main.py b/apps/ai-sre-assistant/app/main.py index 334724f..00e950f 100644 --- a/apps/ai-sre-assistant/app/main.py +++ b/apps/ai-sre-assistant/app/main.py @@ -91,17 +91,18 @@ def summarize_incident(request: SummarizeIncidentRequest) -> dict[str, Any]: if request.use_llm: llm_config = load_config() response["llm_cost_controls"] = cost_controls_summary(llm_config) - llm_analysis, llm_notice = analyze_with_llm( + llm_result = analyze_with_llm( question=question, logs=[], rule_based_analysis=response, config=llm_config, ) - if llm_analysis: + response["llm_telemetry"] = llm_result.telemetry + if llm_result.analysis: response["analysis_mode"] = "llm" - response["llm_analysis"] = llm_analysis - if llm_notice: - response["llm_notice"] = llm_notice + response["llm_analysis"] = llm_result.analysis + if llm_result.notice: + response["llm_notice"] = llm_result.notice return redact_data(response) @@ -123,17 +124,18 @@ def _analyze(question: str, max_lines: int, use_llm: bool) -> dict[str, Any]: if use_llm: llm_config = load_config() response["llm_cost_controls"] = cost_controls_summary(llm_config) - llm_analysis, llm_notice = analyze_with_llm( + llm_result = analyze_with_llm( question=question, logs=logs, rule_based_analysis=rule_based, config=llm_config, ) - if llm_analysis: + response["llm_telemetry"] = llm_result.telemetry + if llm_result.analysis: response["analysis_mode"] = "llm" - response["llm_analysis"] = llm_analysis - if llm_notice: - response["llm_notice"] = llm_notice + response["llm_analysis"] = llm_result.analysis + if llm_result.notice: + response["llm_notice"] = llm_result.notice return redact_data(response) diff --git a/apps/ai-sre-assistant/tests/test_llm_cost_controls.py b/apps/ai-sre-assistant/tests/test_llm_cost_controls.py index 658b697..0fa1cba 100644 --- a/apps/ai-sre-assistant/tests/test_llm_cost_controls.py +++ b/apps/ai-sre-assistant/tests/test_llm_cost_controls.py @@ -1,3 +1,5 @@ +import json + from fastapi.testclient import TestClient from app.llm import LLMConfig, analyze_with_llm, build_user_prompt, cost_controls_summary, load_config @@ -48,15 +50,19 @@ def test_llm_prompt_uses_recent_configured_log_window(): assert "older-log" not in prompt -def test_llm_prompt_respects_prompt_character_budget(monkeypatch): +def test_llm_prompt_respects_budget_and_reports_usage(monkeypatch): captured = {} + clock = iter([10.0, 10.125]) class FakeResponse: def raise_for_status(self): return None def json(self): - return {"choices": [{"message": {"content": "bounded analysis"}}]} + return { + "choices": [{"message": {"content": "bounded analysis"}}], + "usage": {"prompt_tokens": 120, "completion_tokens": 30, "total_tokens": 150}, + } class FakeClient: def __init__(self, timeout): @@ -73,6 +79,7 @@ def post(self, url, headers, json): return FakeResponse() monkeypatch.setattr("app.llm.httpx.Client", FakeClient) + monkeypatch.setattr("app.llm.time.perf_counter", lambda: next(clock)) config = LLMConfig( provider="openai", api_key="provider-key", @@ -82,19 +89,80 @@ def post(self, url, headers, json): max_prompt_chars=350, ) - result, notice = analyze_with_llm( + result = analyze_with_llm( question="Investigate the latest failure", logs=[{"message": "x" * 2000}], rule_based_analysis={"summary": "y" * 1000}, config=config, ) - assert result == "bounded analysis" - assert notice == "LLM prompt was limited to 350 characters by cost controls." + assert result.analysis == "bounded analysis" + assert result.notice == "LLM prompt was limited to 350 characters by cost controls." assert len(captured["prompt"]) <= 350 + assert result.telemetry == { + "provider": "openai", + "model": "test-model", + "configured": True, + "attempted": True, + "outcome": "success", + "request_latency_ms": 125.0, + "fallback_used": False, + "fallback_reason": None, + "usage": { + "reported": True, + "input_tokens": 120, + "output_tokens": 30, + "total_tokens": 150, + }, + } + + +def test_llm_failure_reports_safe_fallback_telemetry(monkeypatch): + clock = iter([20.0, 20.025]) + + class FailingClient: + def __init__(self, timeout): + assert timeout == 30 + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, traceback): + return False + + def post(self, url, headers, json): + raise RuntimeError("Bearer provider-secret failed at https://private-provider.invalid") + + monkeypatch.setattr("app.llm.httpx.Client", FailingClient) + monkeypatch.setattr("app.llm.time.perf_counter", lambda: next(clock)) + config = LLMConfig( + provider="private-provider", + api_key="provider-key", + base_url="https://private-provider.invalid/v1", + model="private-model", + ) + result = analyze_with_llm( + question="Investigate password=question-secret", + logs=[{"message": "Bearer log-secret"}], + rule_based_analysis={"summary": "rule-based summary"}, + config=config, + ) -def test_api_response_reports_llm_cost_controls(tmp_path, monkeypatch): + assert result.analysis is None + assert "provider-secret" not in result.notice + assert result.telemetry["outcome"] == "failure" + assert result.telemetry["request_latency_ms"] == 25.0 + assert result.telemetry["fallback_used"] is True + assert result.telemetry["fallback_reason"] == "provider_request_failed" + serialized = json.dumps(result.telemetry) + assert "provider-key" not in serialized + assert "private-provider.invalid" not in serialized + assert "question-secret" not in serialized + assert "log-secret" not in serialized + + +def test_api_response_reports_cost_controls_and_unconfigured_telemetry(tmp_path, monkeypatch): log_file = tmp_path / "demo-service.log" log_file.write_text('{"level":"INFO","message":"ok","status_code":200}\n', encoding="utf-8") monkeypatch.setenv("DEMO_SERVICE_LOG_PATH", str(log_file)) @@ -111,4 +179,38 @@ def test_api_response_reports_llm_cost_controls(tmp_path, monkeypatch): "max_log_entries": 3, "max_prompt_chars": 4096, } + assert body["llm_telemetry"] == { + "provider": "none", + "model": None, + "configured": False, + "attempted": False, + "outcome": "not_configured", + "request_latency_ms": None, + "fallback_used": True, + "fallback_reason": "provider_not_configured", + "usage": { + "reported": False, + "input_tokens": None, + "output_tokens": None, + "total_tokens": None, + }, + } assert "LLM provider is not configured" in body["llm_notice"] + +def test_summarize_incident_reports_same_unconfigured_telemetry(tmp_path, monkeypatch): + log_file = tmp_path / "demo-service.log" + log_file.write_text('{"level":"INFO","message":"ok","status_code":200}\n', encoding="utf-8") + monkeypatch.setenv("DEMO_SERVICE_LOG_PATH", str(log_file)) + monkeypatch.setenv("LLM_PROVIDER", "none") + + response = client.post( + "/summarize-incident", + json={"include_metrics": False, "use_llm": True}, + ) + + assert response.status_code == 200 + telemetry = response.json()["llm_telemetry"] + assert telemetry["provider"] == "none" + assert telemetry["outcome"] == "not_configured" + assert telemetry["attempted"] is False + assert telemetry["fallback_reason"] == "provider_not_configured" diff --git a/apps/ai-sre-assistant/tests/test_redaction.py b/apps/ai-sre-assistant/tests/test_redaction.py index e30dd03..841d989 100644 --- a/apps/ai-sre-assistant/tests/test_redaction.py +++ b/apps/ai-sre-assistant/tests/test_redaction.py @@ -112,16 +112,21 @@ def post(self, url, headers, json): model="test-model", ) - result, notice = analyze_with_llm( + result = analyze_with_llm( question="Investigate password=question-secret", logs=[{"authorization": "Bearer log-secret", "message": "sk-example123456 failed"}], rule_based_analysis={"summary": "access_token=analysis-secret"}, config=config, ) prompt = captured["payload"]["messages"][1]["content"] - - assert result == f"Safe analysis; Bearer {REDACTED}" - assert notice is None + telemetry = json.dumps(result.telemetry) + + assert result.analysis == f"Safe analysis; Bearer {REDACTED}" + assert result.notice is None + assert "provider-secret" not in telemetry + assert "question-secret" not in telemetry + assert "log-secret" not in telemetry + assert "analysis-secret" not in telemetry assert prompt.count(REDACTED) == 4 assert "question-secret" not in prompt assert "log-secret" not in prompt diff --git a/docs/09-roadmap.md b/docs/09-roadmap.md index f7e11bd..183b9e1 100644 --- a/docs/09-roadmap.md +++ b/docs/09-roadmap.md @@ -1,5 +1,7 @@ # Roadmap +This is the canonical technical execution order for AIOps Lab. Weeks 1-4 are the completed learning foundation, Weeks 5-8 are the next measurement cycle, and later infrastructure remains gated by evidence and named operational ownership. + ## Week 1 - Local Learning Lab - Local demo-service. @@ -35,24 +37,79 @@ - Optional advanced serving roadmap: vLLM, Triton, Ray, KServe, and GPU scheduling. - Production-readiness review with local and CI release gates. -## Commercialization +## Commercialization Track + +The [commercialization roadmap](21-commercialization-roadmap.md) runs alongside the technical weeks: audience first, a design-partner wedge second, and paid tiers third. Completing a local technical week does not bypass its customer-discovery or phase-exit gates. + +## Week 5 - Provider Telemetry + +- Day 1: expose a bounded per-call contract for provider/model identity, request latency, token usage, outcome, and deterministic fallback without storing sensitive content. +- Add aggregate counters and latency distributions with bounded labels. +- Define explicit pricing inputs and estimated cost metadata without hard-coding unstable provider prices. +- Join provider usage and cost with evaluation outcomes. +- Calculate cost per successful evaluated analysis. + +See [Provider Telemetry Contract](22-provider-telemetry.md) for the Day 1 API contract, privacy boundary, and remaining-week sequence. + +**Exit gate:** provider usage, reliability, and cost can be compared with evaluation outcomes without storing prompts, incident evidence, credentials, endpoints, or generated content. + +## Week 6 - Evaluation Maturity + +- Expand the corpus with sanitized incidents, adversarial inputs, prompt-injection cases, incorrect-confidence cases, and redaction edge cases. +- Version the corpus, assistant configuration, and acceptance thresholds together. +- Produce machine-readable evaluation results in CI. +- Keep privacy and safety as hard release gates. + +**Exit gate:** a model, prompt, provider, or code change produces a repeatable regression report and cannot bypass required privacy or safety checks. + +## Week 7 - Production Signal Path + +- Standardize structured stdout logs and cross-service correlation fields. +- Add an optional OpenTelemetry Collector path after signal contracts are stable. +- Add a small dashboard and one actionable, owned alert. +- Exercise one incident from alert through evidence, assistant analysis, runbook action, and recovery review. + +**Exit gate:** the project demonstrates a complete symptom-to-recovery workflow without changing the dependency-light quickstart. + +## Week 8 - Provider Versus Private Endpoint Benchmark + +- Run the same evaluation corpus against deterministic, managed-provider, and OpenAI-compatible private endpoints. +- Measure quality, latency, token usage, fallbacks, throughput, and cost per successful evaluated analysis. +- Test representative input sizes, concurrency, and burst behavior. +- Record a build-versus-buy decision before adding GPU infrastructure. + +**Exit gate:** evidence supports continuing with a provider or starting one bounded private-model experiment. + +## Internal Pilot Phase + +Begin only after the Week 5-8 measurement loop works and the commercialization roadmap has reached the design-partner phase with named owners. + +- Authentication, service identity, and role-based access. +- Managed secrets, rotation, artifact pinning, and supply-chain scanning. +- Ingress, TLS, environment separation, and hardened deployment automation. +- Centralized telemetry, retention policies, audit records, quotas, and budgets. +- Initial SLOs, routed alerts, rollback tests, and recovery exercises. +- Private evaluation datasets and controlled evidence access. + +**Exit gate:** a sanitized internal pilot can operate with explicit ownership, access controls, measurable reliability, and a tested rollback path. + +## Advanced Serving Phase - Only If Earned + +- Test one approved model behind an authenticated OpenAI-compatible endpoint. +- Add an optional single-GPU vLLM example only when the Week 8 benchmark or a named deployment requirement justifies it. +- Add GPU scheduling, quotas, utilization telemetry, queue metrics, and out-of-memory recovery tests for a real workload. +- Introduce Ray Serve, Triton, or KServe only when its specific orchestration problem appears. +- Consider enterprise VPC, dedicated, or on-premises packaging after customer demand is validated. -The phased plan for turning the lab into a monetized product — audience, wedge product, paid team tier — is in `21-commercialization-roadmap.md`. +The default project remains deterministic, provider-compatible, laptop-friendly, and GPU-free throughout these phases. -## Later +## Longer-Term Product Backlog -- OpenTelemetry collector. -- Log backend integration. -- Prometheus and Grafana or managed observability. -- Privacy-aware product usage and outcome telemetry. -- Ingress and TLS walkthrough. -- External secrets workflow. -- Horizontal Pod Autoscaling. - Hosted evaluation history and regression alerts. -- Private incident datasets and release gates. -- Model and provider quality/cost comparisons. -- Provider versus self-hosted inference benchmark harness. -- Optional single-GPU vLLM deployment example. -- Enterprise VPC and on-premises deployment path. -- AI assistant guardrails. -- Cloud deployment examples. +- Team collaboration and private incident datasets. +- Audit-ready exports, policy controls, and usage governance. +- Privacy-aware product outcome telemetry. +- Provider and model quality/cost comparisons over time. +- Log, metrics, and trace backend examples using Prometheus, Grafana, compatible open-source components, or managed services. +- Multi-environment and cloud deployment examples. +- Horizontal Pod Autoscaling for measured non-GPU workloads. diff --git a/docs/18-production-observability.md b/docs/18-production-observability.md index 6ebd302..6039eea 100644 --- a/docs/18-production-observability.md +++ b/docs/18-production-observability.md @@ -70,7 +70,7 @@ A future instrumentation pass can use a bounded metric contract like this: | `ai_sre_eval_checks_total` | `suite`, `dimension`, `result`, `assistant_version` | Quality and release-gate trends. | | `ai_sre_provider_tokens_total` | `provider`, `model`, `direction` | Token usage for quality/cost comparisons. | -These names are a proposed contract, not metrics implemented in Day 5. Request IDs and workspace IDs should not be added as metric labels. Per-customer billing or audit attribution belongs in a controlled event ledger with its own access and retention policy. +Week 5 Day 1 implements per-request provider, model, outcome, fallback, latency, and token-usage metadata in API responses. The metric names above remain a proposed aggregate contract. Request IDs and workspace IDs should not be added as metric labels. Per-customer billing or audit attribution belongs in a controlled event ledger with its own access and retention policy. Do not place full prompts, log evidence, model responses, API keys, or incident payloads in metrics. Detailed audit records require separate access, retention, and redaction controls. @@ -181,4 +181,4 @@ The default AIOps Lab setup should remain dependency-light. A production stack b ## What Comes Next -This guide defines the upgrade path; it does not install a production stack into the beginner workflow. Later optional work can add an OpenTelemetry Collector, backend examples, dashboard definitions, alert rules, and provider usage metrics one layer at a time. +This guide defines the upgrade path; it does not install a production stack into the beginner workflow. The [project roadmap](09-roadmap.md) schedules provider telemetry for Week 5, evaluation maturity for Week 6, and the optional OpenTelemetry signal path plus one alert-to-runbook exercise for Week 7. Backend examples and broader dashboard or alert coverage remain later additions driven by measured operating needs. diff --git a/docs/19-advanced-model-serving-roadmap.md b/docs/19-advanced-model-serving-roadmap.md index 1a134d9..499a68a 100644 --- a/docs/19-advanced-model-serving-roadmap.md +++ b/docs/19-advanced-model-serving-roadmap.md @@ -194,14 +194,18 @@ Before committing to a serving platform, require: ## Recommended Implementation Order -1. Capture provider token usage, model identity, latency, outcome, and fallback metrics. -2. Add a provider-versus-self-host benchmark harness using the existing evaluation corpus. -3. Test one approved model behind an authenticated OpenAI-compatible endpoint. -4. Add an optional single-GPU deployment example without changing the default quickstart. -5. Prove quality, reliability, and unit economics under representative load. -6. Add Kubernetes GPU scheduling and observability for a real deployment requirement. -7. Introduce Ray Serve, Triton, or KServe only when its specific orchestration problem appears. -8. Package private deployment, governance, metering, and support around validated customer demand. +The first four steps are the measurement work scheduled for Weeks 5-8 of the [project roadmap](09-roadmap.md). Serving infrastructure begins only when those steps and the [commercialization gates](21-commercialization-roadmap.md) support it: + +1. Capture provider token usage, model identity, latency, outcome, fallback, and cost metadata in Week 5. +2. Expand and version the evaluation corpus in Week 6 so every candidate uses the same privacy and safety gates. +3. Stabilize the optional production signal path and operational exercise in Week 7. +4. Run the provider-versus-private-endpoint benchmark and record the build-versus-buy decision in Week 8. +5. Test one approved model behind an authenticated OpenAI-compatible endpoint only when the decision supports it. +6. Add an optional single-GPU deployment example without changing the default quickstart. +7. Prove quality, reliability, and unit economics under representative load. +8. Add Kubernetes GPU scheduling and observability for a real deployment requirement. +9. Introduce Ray Serve, Triton, or KServe only when its specific orchestration problem appears. +10. Package private deployment, governance, metering, and support around validated customer demand. ## Day 6 Definition Of Done diff --git a/docs/20-production-readiness-review.md b/docs/20-production-readiness-review.md index fc81d93..8ed996b 100644 --- a/docs/20-production-readiness-review.md +++ b/docs/20-production-readiness-review.md @@ -68,15 +68,16 @@ Any failed privacy or safety gate is a no-go. Other exceptions need a named owne ## First 30 Days After This Milestone -The next work should deepen measured contracts before adding infrastructure breadth: +The next four technical weeks deepen measured contracts before adding infrastructure breadth. They match the canonical sequence in the [project roadmap](09-roadmap.md) and do not bypass the audience or design-partner gates in the [commercialization roadmap](21-commercialization-roadmap.md). -1. Capture provider model, latency, token usage, failure, and fallback metadata without storing prompt content. -2. Expand the evaluation corpus with sanitized incidents and version its acceptance thresholds. -3. Add optional OpenTelemetry collection after log, metric, and trace field contracts are stable. -4. Turn one incident walkthrough into an alert-to-runbook exercise with an owner and measurable recovery result. -5. Build a provider-versus-private-endpoint benchmark harness before adding a GPU deployment example. +| Week | Focus | Required outcome | +| --- | --- | --- | +| 5 | Provider telemetry | Start with the privacy-safe per-call contract, then aggregate model, latency, token usage, failure, fallback, and cost metadata and compare it with evaluation outcomes. | +| 6 | Evaluation maturity | Expand and version sanitized and adversarial cases, produce machine-readable CI results, and preserve privacy and safety as hard gates. | +| 7 | Production signal path | Add optional OpenTelemetry collection only after signal contracts are stable, then exercise one owned alert-to-runbook recovery flow. | +| 8 | Provider versus private benchmark | Compare deterministic, provider, and compatible private endpoints with the same corpus and record a build-versus-buy decision. | -vLLM, Triton, Ray Serve, KServe, GPU scheduling, multi-tenancy, and paid platform features stay behind demonstrated demand and the adoption gates already documented in the advanced serving roadmap. +Internal-pilot controls begin only after this measurement loop works and customer discovery supports a design-partner deployment. vLLM, Triton, Ray Serve, KServe, GPU scheduling, multi-tenancy, and paid platform features stay behind demonstrated demand and the adoption gates in the [advanced model serving roadmap](19-advanced-model-serving-roadmap.md). ## Day 7 Definition Of Done diff --git a/docs/21-commercialization-roadmap.md b/docs/21-commercialization-roadmap.md index 4ab6a60..4c85397 100644 --- a/docs/21-commercialization-roadmap.md +++ b/docs/21-commercialization-roadmap.md @@ -58,7 +58,7 @@ The wedge thesis: teams will adopt an incident analyst they can hold to a measur | Milestone | Definition of done | | --- | --- | | First real connector | Read-only connector to one real log source (CloudWatch Logs, Loki, or Datadog), honoring the existing bounded-query and redaction policies. The shared-file path remains the learning default. | -| Provider metering | Model identity, latency, token usage, fallback outcome, and cost per successful analysis captured as metrics without storing prompt content (item 1 from `docs/20-production-readiness-review.md`). | +| Provider metering | Build on the Week 5 per-request contract: aggregate model identity, latency, token usage, fallback outcome, and cost per successful evaluated analysis without storing prompt or incident content. The milestone is incomplete until design-partner usage is measurable. | | Private incident sets | A team can record sanitized incidents from their own systems and run the evaluation corpus against them in CI, blocking regressions. | | Minimum viable trust | OIDC/SSO authentication, per-user audit log of evidence access, and a documented single-tenant deployment using the existing Kubernetes manifests. | | Design partners | Two or three teams running the assistant against sanitized real data weekly, with a shared feedback channel and written usage notes. | diff --git a/docs/22-provider-telemetry.md b/docs/22-provider-telemetry.md new file mode 100644 index 0000000..b56eda3 --- /dev/null +++ b/docs/22-provider-telemetry.md @@ -0,0 +1,112 @@ +# Provider Telemetry Contract + +Week 5, Day 1 establishes the privacy-safe metadata contract for optional LLM calls. + +The assistant already bounded prompts and fell back to deterministic analysis. The missing foundation was a consistent way to answer: which provider path was selected, was a request attempted, did it succeed, how long did it take, did the provider report token usage, and did the deterministic fallback protect the user? + +## API Contract + +When `use_llm=true`, `/ask`, `/analyze/logs`, and `/summarize-incident` include `llm_telemetry` alongside the existing analysis and cost-control fields. + +```json +{ + "llm_telemetry": { + "provider": "openai", + "model": "approved-model", + "configured": true, + "attempted": true, + "outcome": "success", + "request_latency_ms": 842.37, + "fallback_used": false, + "fallback_reason": null, + "usage": { + "reported": true, + "input_tokens": 420, + "output_tokens": 96, + "total_tokens": 516 + } + } +} +``` + +The contract is per request. It is not yet a persistent usage ledger, billing record, Prometheus metric implementation, or cost estimate. + +## Field Semantics + +| Field | Meaning | +| --- | --- | +| `provider` | Bounded, redacted provider label from configuration. It never contains the base URL. | +| `model` | Bounded, redacted model label when the provider is configured; otherwise `null`. | +| `configured` | Whether the selected provider has the minimum required configuration. | +| `attempted` | Whether the assistant attempted a network request to the provider. | +| `outcome` | One of `success`, `failure`, or `not_configured`. | +| `request_latency_ms` | Provider request duration measured with a monotonic clock; `null` when no request was attempted. | +| `fallback_used` | Whether the final response stayed on the deterministic rule-based path. | +| `fallback_reason` | `provider_not_configured`, `provider_request_failed`, or `null`. | +| `usage.reported` | Whether the provider returned usable token counts. | +| `usage.input_tokens` | Normalized `prompt_tokens`, or `null` when unavailable. | +| `usage.output_tokens` | Normalized `completion_tokens`, or `null` when unavailable. | +| `usage.total_tokens` | Provider total, or the sum of valid input and output counts when both exist. | + +Unknown, negative, boolean, or malformed token values are treated as unavailable. Providers may omit usage; omission does not turn a successful analysis into a failure. + +## Outcomes And Fallbacks + +| Situation | Outcome | Attempted | Fallback | +| --- | --- | --- | --- | +| `LLM_PROVIDER=none` or incomplete configuration | `not_configured` | No | Yes: `provider_not_configured` | +| Valid response with non-empty content | `success` | Yes | No | +| Network, HTTP, parsing, or empty-content failure | `failure` | Yes | Yes: `provider_request_failed` | +| Request sets `use_llm=false` | No LLM telemetry block | No | The deterministic path was explicitly selected, not used as a provider fallback. | + +## Privacy Boundary + +Provider telemetry may contain only bounded operational metadata. + +Allowed: + +- Provider and model labels. +- Configuration, attempt, outcome, and fallback state. +- Request latency. +- Numeric token counts. + +Forbidden: + +- Questions or prompts. +- Log, metric, trace, or incident evidence. +- API keys, authorization headers, or credentials. +- Provider base URLs. +- Model output or failure payloads. +- User, request, workspace, or incident identifiers. + +Tests inject secrets into questions, logs, credentials, provider output, and failure messages and verify that those values never enter the telemetry object. The final API response still passes through the existing defense-in-depth redaction boundary. + +## Customer And Product Value + +This contract supports the commercialization roadmap without creating a surveillance or billing system prematurely: + +- Design partners can see whether enrichment succeeded or silently fell back. +- Teams can compare provider latency and token usage against evaluated quality later. +- Product decisions can use cost per successful analysis instead of token price alone. +- Private endpoints and managed providers can use the same normalized outcome contract. + +Do not attach customer identifiers to future Prometheus labels. Per-team billing, audit attribution, and durable usage history require an access-controlled event ledger with explicit retention. + +## Day 1 Definition Of Done + +- Provider and model identity are exposed as bounded, redacted labels. +- Success, failure, unconfigured state, and deterministic fallback are distinguishable. +- Provider request latency uses a monotonic clock. +- OpenAI-compatible token usage is normalized when present and honest when absent. +- No prompt, evidence, credential, endpoint URL, or generated output enters telemetry. +- Both LLM-enabled API flows expose the same contract. +- Success, failure, fallback, usage, latency, and privacy behavior have deterministic tests. + +## Remaining Week 5 Sequence + +1. Add bounded aggregate counters and latency distributions without high-cardinality labels. +2. Define explicit model pricing inputs and estimated cost metadata without hard-coding unstable prices. +3. Join provider outcomes and cost with evaluation results to calculate cost per successful evaluated analysis. +4. Add a local comparison report across deterministic and configured provider paths. +5. Exercise provider failure and fallback behavior end to end. +6. Close the week with an exit-gate review against the technical and commercialization roadmaps. diff --git a/docs/build-log.md b/docs/build-log.md index 959dde5..86bd66d 100644 --- a/docs/build-log.md +++ b/docs/build-log.md @@ -666,3 +666,51 @@ What comes next: - Add an optional telemetry collector only after signal contracts are stable. - Exercise one alert and runbook path end to end. - Benchmark provider and private endpoints before introducing GPU infrastructure. + +## Week 5, Day 1 - Provider Telemetry Contract + +Today I started the Week 5 measurement cycle by making optional provider behavior visible without recording sensitive incident content. + +Week 4 established redaction, prompt bounds, deterministic evaluation, and production-readiness gates. Week 5 starts with the metadata needed to compare provider reliability, quality, and cost honestly. + +What changed: + +- Added a structured LLM call result with analysis, notice, and bounded telemetry. +- Added provider and model labels without exposing the provider base URL. +- Added configuration, attempt, success, failure, and fallback state. +- Measured provider request latency with a monotonic clock. +- Normalized OpenAI-compatible prompt, completion, and total token usage when reported. +- Kept missing or malformed usage honest with explicit `null` values and `reported=false`. +- Exposed the same `llm_telemetry` contract from log analysis, questions, and incident summaries. +- Added tests for successful usage, latency, provider failure, unconfigured fallback, both API paths, and telemetry privacy. +- Added a provider telemetry guide with field semantics, forbidden data, customer value, and the remaining Week 5 sequence. +- Restored the canonical Weeks 5-8 technical roadmap while preserving the audience and design-partner gates in the commercialization roadmap. + +Why this matters: + +A provider call should not be a black box. Teams need to know whether enrichment ran, whether it failed safely, how long it took, and whether the provider reported usage. Those facts are the foundation for comparing quality and cost later. + +The metadata boundary matters just as much as the fields. Prompt text, incident evidence, credentials, endpoint URLs, generated output, and customer identifiers do not belong in provider telemetry. + +Customer and monetization value: + +- Design partners can distinguish successful enrichment from silent deterministic fallback. +- Teams can later compare provider latency and token usage with evaluated quality. +- Cost can be measured per successful analysis instead of per token in isolation. +- Managed and private OpenAI-compatible endpoints share the same outcome contract. +- Completing this local contract does not skip the commercialization roadmap's discovery or design-partner gates. + +Lessons learned: + +- Missing usage metadata is not zero usage; it is unknown and should be represented honestly. +- Provider failure and provider-not-configured are different operational outcomes. +- A monotonic clock is the right boundary for request duration. +- Useful telemetry can remain small, bounded, and free of incident content. +- Per-request metadata should come before aggregate metrics, billing ledgers, or dashboards. + +What comes next: + +- Add bounded aggregate counters and latency distributions. +- Define explicit pricing inputs without hard-coding unstable provider prices. +- Join provider usage and cost with evaluation outcomes. +- Calculate and report cost per successful evaluated analysis.