Week 5 Day 1: add provider telemetry contract

README.md

@@ -99,6 +99,8 @@ For the Week 4 release gates, readiness verdict, and next-stage priorities, see
 
 For the phased commercialization roadmap with milestones and success metrics, see `docs/21-commercialization-roadmap.md`.
 
+For the Week 5 provider telemetry contract and privacy boundary, see `docs/22-provider-telemetry.md`.
+
 Ask the assistant directly:
 
 ```bash
@@ -144,6 +146,7 @@ AIOps-Lab/
     19-advanced-model-serving-roadmap.md
     20-production-readiness-review.md
     21-commercialization-roadmap.md
+    22-provider-telemetry.md
   infra/                 # Docker, Kubernetes, and Terraform starter notes
     k8s/                 # kind-first Kubernetes manifests and walkthrough
   scripts/               # Local traffic and log helper scripts
@@ -174,6 +177,12 @@ You should see the assistant report intentionally generated 500s, latency spikes
 - Week 2: observability basics, metrics, dashboards, structured logging, incident examples.
 - Week 3: Kubernetes manifests, operations, config/secrets, probes/resources, incident debugging, production next steps.
 - Week 4: security hardening, cost optimization, evaluation, production observability, advanced serving decisions, and a release-gated production-readiness review.
+- Week 5: privacy-safe provider identity, latency, usage, outcome, fallback, and cost telemetry.
+- Week 6: a larger versioned evaluation corpus with machine-readable CI regression reports.
+- Week 7: an optional OpenTelemetry signal path plus one owned alert-to-runbook exercise.
+- Week 8: a provider-versus-private-endpoint benchmark and an evidence-backed build-versus-buy decision.
+
+The technical weeks run alongside the commercialization phases; they do not bypass audience, design-partner, or willingness-to-pay gates. See `docs/09-roadmap.md` and `docs/21-commercialization-roadmap.md`.
 
 ## Build In Public
 

apps/ai-sre-assistant/README.md

@@ -80,6 +80,14 @@ API responses that attempt LLM enrichment include `llm_cost_controls` so callers
 
 See `../../docs/16-cost-optimization.md` for the Day 3 cost optimization guide.
 
+## Provider Telemetry
+
+When `use_llm=true`, LLM-enabled API responses include `llm_telemetry` with bounded provider and model labels, configuration and attempt state, success or fallback outcome, request latency, and normalized token usage when the provider reports it.
+
+The telemetry object never includes prompts, incident evidence, credentials, provider base URLs, generated output, or user and incident identifiers. It is per-request metadata, not yet a persistent usage ledger, billing record, cost estimate, or Prometheus metrics implementation.
+
+See `../../docs/22-provider-telemetry.md` for field semantics, privacy rules, failure behavior, and the remaining Week 5 sequence.
+
 ## Evaluation
 
 The assistant includes a deterministic incident corpus and a five-part quality rubric. Run it locally from this directory:

apps/ai-sre-assistant/app/llm.py

@@ -1,5 +1,6 @@
 import json
 import os
+import time
 from dataclasses import dataclass
 from typing import Any
 
@@ -12,6 +13,7 @@
 DEFAULT_LLM_MAX_LOG_ENTRIES = 50
 DEFAULT_LLM_MAX_PROMPT_CHARS = 12000
 TRUNCATION_MARKER = "\n...[truncated by LLM cost controls]\n"
+MAX_TELEMETRY_LABEL_CHARS = 100
 
 
 @dataclass(frozen=True)
@@ -24,6 +26,13 @@ class LLMConfig:
     max_prompt_chars: int = DEFAULT_LLM_MAX_PROMPT_CHARS
 
 
+@dataclass(frozen=True)
+class LLMCallResult:
+    analysis: str | None
+    notice: str | None
+    telemetry: dict[str, Any]
+
+
 def load_config() -> LLMConfig:
     provider = os.getenv("LLM_PROVIDER", "none").strip().lower()
     default_base_url = "http://localhost:11434/v1" if provider == "ollama" else "https://api.openai.com/v1"
@@ -57,10 +66,21 @@ def analyze_with_llm(
     logs: list[dict[str, Any]],
     rule_based_analysis: dict[str, Any],
     config: LLMConfig | None = None,
-) -> tuple[str | None, str | None]:
+) -> LLMCallResult:
     config = config or load_config()
     if not is_configured(config):
-        return None, "LLM provider is not configured; using rule-based analysis."
+        return LLMCallResult(
+            analysis=None,
+            notice="LLM provider is not configured; using rule-based analysis.",
+            telemetry=_telemetry(
+                config=config,
+                configured=False,
+                attempted=False,
+                outcome="not_configured",
+                fallback_used=True,
+                fallback_reason="provider_not_configured",
+            ),
+        )
 
     user_prompt, prompt_was_truncated = build_user_prompt(
         question=question,
@@ -87,18 +107,47 @@ def analyze_with_llm(
         "temperature": 0.2,
     }
 
+    request_started = time.perf_counter()
     try:
         with httpx.Client(timeout=30) as client:
             response = client.post(f"{config.base_url}/chat/completions", headers=headers, json=payload)
             response.raise_for_status()
             data = response.json()
             content = data["choices"][0]["message"]["content"]
-            return redact_text(str(content).strip()), cost_notice
+            analysis = redact_text(str(content).strip())
+            if not analysis:
+                raise ValueError("LLM response content was empty")
+            return LLMCallResult(
+                analysis=analysis,
+                notice=cost_notice,
+                telemetry=_telemetry(
+                    config=config,
+                    configured=True,
+                    attempted=True,
+                    outcome="success",
+                    fallback_used=False,
+                    fallback_reason=None,
+                    request_latency_ms=_elapsed_ms(request_started),
+                    usage=data.get("usage"),
+                ),
+            )
     except Exception as exc:
         failure_notice = f"LLM request failed; using rule-based analysis. Error: {redact_text(str(exc))}"
         if cost_notice:
             failure_notice = f"{cost_notice} {failure_notice}"
-        return None, failure_notice
+        return LLMCallResult(
+            analysis=None,
+            notice=failure_notice,
+            telemetry=_telemetry(
+                config=config,
+                configured=True,
+                attempted=True,
+                outcome="failure",
+                fallback_used=True,
+                fallback_reason="provider_request_failed",
+                request_latency_ms=_elapsed_ms(request_started),
+            ),
+        )
 
 
 def build_user_prompt(
@@ -154,3 +203,57 @@ def _truncate_text(value: str, max_chars: int) -> tuple[str, bool]:
         return value[:max_chars], True
     return value[: max_chars - len(TRUNCATION_MARKER)].rstrip() + TRUNCATION_MARKER, True
 
+
+def _telemetry(
+    config: LLMConfig,
+    *,
+    configured: bool,
+    attempted: bool,
+    outcome: str,
+    fallback_used: bool,
+    fallback_reason: str | None,
+    request_latency_ms: float | None = None,
+    usage: Any = None,
+) -> dict[str, Any]:
+    return {
+        "provider": _bounded_label(config.provider or "none"),
+        "model": _bounded_label(config.model) if configured else None,
+        "configured": configured,
+        "attempted": attempted,
+        "outcome": outcome,
+        "request_latency_ms": request_latency_ms,
+        "fallback_used": fallback_used,
+        "fallback_reason": fallback_reason,
+        "usage": _normalize_usage(usage),
+    }
+
+
+def _normalize_usage(usage: Any) -> dict[str, int | bool | None]:
+    if not isinstance(usage, dict):
+        usage = {}
+
+    input_tokens = _non_negative_int(usage.get("prompt_tokens"))
+    output_tokens = _non_negative_int(usage.get("completion_tokens"))
+    total_tokens = _non_negative_int(usage.get("total_tokens"))
+    if total_tokens is None and input_tokens is not None and output_tokens is not None:
+        total_tokens = input_tokens + output_tokens
+
+    return {
+        "reported": any(value is not None for value in (input_tokens, output_tokens, total_tokens)),
+        "input_tokens": input_tokens,
+        "output_tokens": output_tokens,
+        "total_tokens": total_tokens,
+    }
+
+
+def _non_negative_int(value: Any) -> int | None:
+    return value if isinstance(value, int) and not isinstance(value, bool) and value >= 0 else None
+
+
+def _bounded_label(value: str) -> str:
+    return redact_text(str(value).strip())[:MAX_TELEMETRY_LABEL_CHARS]
+
+
+def _elapsed_ms(started: float) -> float:
+    return round(max(0.0, (time.perf_counter() - started) * 1000), 2)
+

apps/ai-sre-assistant/app/main.py

@@ -91,17 +91,18 @@ def summarize_incident(request: SummarizeIncidentRequest) -> dict[str, Any]:
     if request.use_llm:
         llm_config = load_config()
         response["llm_cost_controls"] = cost_controls_summary(llm_config)
-        llm_analysis, llm_notice = analyze_with_llm(
+        llm_result = analyze_with_llm(
             question=question,
             logs=[],
             rule_based_analysis=response,
             config=llm_config,
         )
-        if llm_analysis:
+        response["llm_telemetry"] = llm_result.telemetry
+        if llm_result.analysis:
             response["analysis_mode"] = "llm"
-            response["llm_analysis"] = llm_analysis
-        if llm_notice:
-            response["llm_notice"] = llm_notice
+            response["llm_analysis"] = llm_result.analysis
+        if llm_result.notice:
+            response["llm_notice"] = llm_result.notice
 
     return redact_data(response)
 
@@ -123,17 +124,18 @@ def _analyze(question: str, max_lines: int, use_llm: bool) -> dict[str, Any]:
     if use_llm:
         llm_config = load_config()
         response["llm_cost_controls"] = cost_controls_summary(llm_config)
-        llm_analysis, llm_notice = analyze_with_llm(
+        llm_result = analyze_with_llm(
             question=question,
             logs=logs,
             rule_based_analysis=rule_based,
             config=llm_config,
         )
-        if llm_analysis:
+        response["llm_telemetry"] = llm_result.telemetry
+        if llm_result.analysis:
             response["analysis_mode"] = "llm"
-            response["llm_analysis"] = llm_analysis
-        if llm_notice:
-            response["llm_notice"] = llm_notice
+            response["llm_analysis"] = llm_result.analysis
+        if llm_result.notice:
+            response["llm_notice"] = llm_result.notice
 
     return redact_data(response)