From 7dd41792a1feed96bf50f74512ae1268b4b1d920 Mon Sep 17 00:00:00 2001 From: Utkarsh Pandey Date: Wed, 22 Jul 2026 11:02:21 -0400 Subject: [PATCH] feat: add provider aggregate metrics --- README.md | 3 +- apps/ai-sre-assistant/README.md | 5 +- apps/ai-sre-assistant/app/llm.py | 5 +- apps/ai-sre-assistant/app/main.py | 8 +- apps/ai-sre-assistant/app/provider_metrics.py | 189 ++++++++++++++++++ .../tests/test_provider_metrics.py | 100 +++++++++ docs/09-roadmap.md | 6 +- docs/18-production-observability.md | 4 +- docs/22-provider-telemetry.md | 56 +++++- docs/build-log.md | 40 ++++ 10 files changed, 399 insertions(+), 17 deletions(-) create mode 100644 apps/ai-sre-assistant/app/provider_metrics.py create mode 100644 apps/ai-sre-assistant/tests/test_provider_metrics.py diff --git a/README.md b/README.md index ae5f36b..6b0c3df 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ Useful local URLs: - Demo metrics: `http://localhost:8000/metrics` - AI SRE Assistant: `http://localhost:8001` - AI SRE health: `http://localhost:8001/health` +- AI SRE provider metrics: `http://localhost:8001/metrics` For the kind-based Kubernetes walkthrough, see `infra/k8s/README.md`. @@ -99,7 +100,7 @@ For the Week 4 release gates, readiness verdict, and next-stage priorities, see For the phased commercialization roadmap with milestones and success metrics, see `docs/21-commercialization-roadmap.md`. -For the Week 5 provider telemetry contract and privacy boundary, see `docs/22-provider-telemetry.md`. +For the Week 5 provider telemetry contract, aggregate metrics, and privacy boundary, see `docs/22-provider-telemetry.md`. Ask the assistant directly: diff --git a/apps/ai-sre-assistant/README.md b/apps/ai-sre-assistant/README.md index 1738798..b8ce069 100644 --- a/apps/ai-sre-assistant/README.md +++ b/apps/ai-sre-assistant/README.md @@ -7,6 +7,7 @@ It can use an OpenAI-compatible LLM provider for richer analysis. If no provider ## API - `GET /health` +- `GET /metrics` - `POST /analyze/logs` - `POST /analyze/metrics` - `POST /ask` @@ -84,7 +85,9 @@ See `../../docs/16-cost-optimization.md` for the Day 3 cost optimization guide. When `use_llm=true`, LLM-enabled API responses include `llm_telemetry` with bounded provider and model labels, configuration and attempt state, success or fallback outcome, request latency, and normalized token usage when the provider reports it. -The telemetry object never includes prompts, incident evidence, credentials, provider base URLs, generated output, or user and incident identifiers. It is per-request metadata, not yet a persistent usage ledger, billing record, cost estimate, or Prometheus metrics implementation. +The telemetry object never includes prompts, incident evidence, credentials, provider base URLs, generated output, or user and incident identifiers. It is per-request metadata, not a persistent usage ledger, billing record, or cost estimate. + +`GET /metrics` exposes privacy-safe in-memory aggregates for analysis outcomes, provider request latency, deterministic fallbacks, and provider-reported input/output tokens. Labels are limited to the configured provider/model and fixed outcome, reason, and direction enums. These process-local counters reset when the assistant restarts. See `../../docs/22-provider-telemetry.md` for field semantics, privacy rules, failure behavior, and the remaining Week 5 sequence. diff --git a/apps/ai-sre-assistant/app/llm.py b/apps/ai-sre-assistant/app/llm.py index a7b6f22..a095d1e 100644 --- a/apps/ai-sre-assistant/app/llm.py +++ b/apps/ai-sre-assistant/app/llm.py @@ -7,6 +7,7 @@ import httpx from app.prompts import ANALYSIS_INSTRUCTIONS, SYSTEM_PROMPT +from app.provider_metrics import PROVIDER_METRICS from app.redaction import redact_data, redact_text @@ -215,7 +216,7 @@ def _telemetry( request_latency_ms: float | None = None, usage: Any = None, ) -> dict[str, Any]: - return { + telemetry = { "provider": _bounded_label(config.provider or "none"), "model": _bounded_label(config.model) if configured else None, "configured": configured, @@ -226,6 +227,8 @@ def _telemetry( "fallback_reason": fallback_reason, "usage": _normalize_usage(usage), } + PROVIDER_METRICS.observe(telemetry) + return telemetry def _normalize_usage(usage: Any) -> dict[str, int | bool | None]: diff --git a/apps/ai-sre-assistant/app/main.py b/apps/ai-sre-assistant/app/main.py index 00e950f..53c9e9b 100644 --- a/apps/ai-sre-assistant/app/main.py +++ b/apps/ai-sre-assistant/app/main.py @@ -1,6 +1,6 @@ from typing import Any -from fastapi import FastAPI +from fastapi import FastAPI, Response from pydantic import BaseModel, Field from app.analyzer import analyze_logs @@ -8,6 +8,7 @@ from app.log_reader import get_log_path, read_recent_logs from app.metrics_analyzer import analyze_metrics, combined_incident_analysis from app.metrics_reader import fetch_metrics_text, get_metrics_url, parse_prometheus_text +from app.provider_metrics import PROVIDER_METRICS from app.redaction import redact_data, redact_text @@ -47,6 +48,11 @@ def health() -> dict[str, str]: return {"status": "healthy", "service": "ai-sre-assistant"} +@app.get("/metrics") +def metrics() -> Response: + return Response(content=PROVIDER_METRICS.render_prometheus(), media_type="text/plain") + + @app.post("/analyze/logs") def analyze_recent_logs(request: AnalyzeLogsRequest) -> dict[str, Any]: return _analyze(question="What happened recently in demo-service logs?", max_lines=request.max_lines, use_llm=request.use_llm) diff --git a/apps/ai-sre-assistant/app/provider_metrics.py b/apps/ai-sre-assistant/app/provider_metrics.py new file mode 100644 index 0000000..162a88b --- /dev/null +++ b/apps/ai-sre-assistant/app/provider_metrics.py @@ -0,0 +1,189 @@ +from collections import Counter, defaultdict +from math import isfinite +from threading import Lock +from typing import Any + +from app.redaction import redact_text + + +PROVIDER_LATENCY_BUCKETS_SECONDS = ( + 0.05, + 0.1, + 0.25, + 0.5, + 1.0, + 2.5, + 5.0, + 10.0, + 30.0, + float("inf"), +) +MAX_METRIC_LABEL_CHARS = 100 +VALID_OUTCOMES = {"success", "failure", "not_configured"} +VALID_FALLBACK_REASONS = {"provider_not_configured", "provider_request_failed"} + + +class ProviderMetricsRegistry: + """Small in-memory provider registry with a fixed, privacy-safe label contract.""" + + def __init__(self) -> None: + self._lock = Lock() + self._analyses: Counter[tuple[str, str, str]] = Counter() + self._latency_buckets: Counter[tuple[str, str, str, float]] = Counter() + self._latency_count: Counter[tuple[str, str, str]] = Counter() + self._latency_sum: defaultdict[tuple[str, str, str], float] = defaultdict(float) + self._fallbacks: Counter[tuple[str, str, str]] = Counter() + self._tokens: Counter[tuple[str, str, str]] = Counter() + + def reset(self) -> None: + with self._lock: + self._analyses.clear() + self._latency_buckets.clear() + self._latency_count.clear() + self._latency_sum.clear() + self._fallbacks.clear() + self._tokens.clear() + + def observe(self, telemetry: dict[str, Any]) -> None: + provider = _bounded_label(telemetry.get("provider"), default="unknown") + model = _bounded_label(telemetry.get("model"), default="none") + outcome = telemetry.get("outcome") + outcome = outcome if outcome in VALID_OUTCOMES else "unknown" + + with self._lock: + self._analyses[(provider, model, outcome)] += 1 + + latency_seconds = _latency_seconds(telemetry) + if latency_seconds is not None: + latency_key = (provider, model, outcome) + self._latency_count[latency_key] += 1 + self._latency_sum[latency_key] += latency_seconds + for bucket in PROVIDER_LATENCY_BUCKETS_SECONDS: + if latency_seconds <= bucket: + self._latency_buckets[(*latency_key, bucket)] += 1 + + fallback_reason = telemetry.get("fallback_reason") + if telemetry.get("fallback_used") is True: + reason = ( + fallback_reason + if fallback_reason in VALID_FALLBACK_REASONS + else "other" + ) + self._fallbacks[(provider, model, reason)] += 1 + + usage = telemetry.get("usage") + if isinstance(usage, dict): + for direction, field in ( + ("input", "input_tokens"), + ("output", "output_tokens"), + ): + value = usage.get(field) + if ( + isinstance(value, int) + and not isinstance(value, bool) + and value >= 0 + ): + self._tokens[(provider, model, direction)] += value + + def render_prometheus(self) -> str: + with self._lock: + analyses = self._analyses.copy() + latency_buckets = self._latency_buckets.copy() + latency_count = self._latency_count.copy() + latency_sum = dict(self._latency_sum) + fallbacks = self._fallbacks.copy() + tokens = self._tokens.copy() + + lines = [ + "# HELP ai_sre_provider_analyses_total LLM enrichment selections by provider, model, and outcome.", + "# TYPE ai_sre_provider_analyses_total counter", + ] + for (provider, model, outcome), count in sorted(analyses.items()): + labels = _labels(provider=provider, model=model, outcome=outcome) + lines.append(f"ai_sre_provider_analyses_total{{{labels}}} {count}") + + lines.extend( + [ + "# HELP ai_sre_provider_request_duration_seconds Attempted provider request duration in seconds.", + "# TYPE ai_sre_provider_request_duration_seconds histogram", + ] + ) + for provider, model, outcome in sorted(latency_count): + for bucket in PROVIDER_LATENCY_BUCKETS_SECONDS: + labels = _labels( + provider=provider, + model=model, + outcome=outcome, + le=_format_bucket(bucket), + ) + count = latency_buckets[(provider, model, outcome, bucket)] + lines.append( + f"ai_sre_provider_request_duration_seconds_bucket{{{labels}}} {count}" + ) + + base_labels = _labels(provider=provider, model=model, outcome=outcome) + lines.append( + f"ai_sre_provider_request_duration_seconds_sum{{{base_labels}}} " + f"{latency_sum[(provider, model, outcome)]:.6f}" + ) + lines.append( + f"ai_sre_provider_request_duration_seconds_count{{{base_labels}}} " + f"{latency_count[(provider, model, outcome)]}" + ) + + lines.extend( + [ + "# HELP ai_sre_provider_fallbacks_total Deterministic provider fallbacks by bounded reason.", + "# TYPE ai_sre_provider_fallbacks_total counter", + ] + ) + for (provider, model, reason), count in sorted(fallbacks.items()): + labels = _labels(provider=provider, model=model, reason=reason) + lines.append(f"ai_sre_provider_fallbacks_total{{{labels}}} {count}") + + lines.extend( + [ + "# HELP ai_sre_provider_tokens_total Provider-reported tokens by input or output direction.", + "# TYPE ai_sre_provider_tokens_total counter", + ] + ) + for (provider, model, direction), count in sorted(tokens.items()): + labels = _labels(provider=provider, model=model, direction=direction) + lines.append(f"ai_sre_provider_tokens_total{{{labels}}} {count}") + + return "\n".join(lines) + "\n" + + +def _latency_seconds(telemetry: dict[str, Any]) -> float | None: + if telemetry.get("attempted") is not True: + return None + latency_ms = telemetry.get("request_latency_ms") + if not isinstance(latency_ms, (int, float)) or isinstance(latency_ms, bool): + return None + if not isfinite(latency_ms) or latency_ms < 0: + return None + return latency_ms / 1000 + + +def _bounded_label(value: Any, *, default: str) -> str: + if value is None: + return default + label = redact_text(str(value).strip())[:MAX_METRIC_LABEL_CHARS] + return label or default + + +def _format_bucket(bucket: float) -> str: + if bucket == float("inf"): + return "+Inf" + return f"{bucket:g}" + + +def _labels(**labels: str) -> str: + return ",".join(f'{key}="{_escape_label(value)}"' for key, value in labels.items()) + + +def _escape_label(value: str) -> str: + return str(value).replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"') + + +PROVIDER_METRICS = ProviderMetricsRegistry() diff --git a/apps/ai-sre-assistant/tests/test_provider_metrics.py b/apps/ai-sre-assistant/tests/test_provider_metrics.py new file mode 100644 index 0000000..4c24283 --- /dev/null +++ b/apps/ai-sre-assistant/tests/test_provider_metrics.py @@ -0,0 +1,100 @@ +from fastapi.testclient import TestClient + +from app.main import app +from app.provider_metrics import ProviderMetricsRegistry + + +def test_provider_metrics_aggregate_outcomes_latency_fallbacks_and_tokens(): + metrics = ProviderMetricsRegistry() + + metrics.observe( + { + "provider": "openai", + "model": "approved-model", + "attempted": True, + "outcome": "success", + "request_latency_ms": 125.0, + "fallback_used": False, + "fallback_reason": None, + "usage": {"input_tokens": 120, "output_tokens": 30}, + } + ) + metrics.observe( + { + "provider": "openai", + "model": "approved-model", + "attempted": True, + "outcome": "failure", + "request_latency_ms": 2500.0, + "fallback_used": True, + "fallback_reason": "provider_request_failed", + "usage": {}, + } + ) + + rendered = metrics.render_prometheus() + + assert ( + 'ai_sre_provider_analyses_total{provider="openai",model="approved-model",outcome="success"} 1' + in rendered + ) + assert ( + 'ai_sre_provider_analyses_total{provider="openai",model="approved-model",outcome="failure"} 1' + in rendered + ) + assert ( + 'ai_sre_provider_request_duration_seconds_bucket{provider="openai",model="approved-model",' + 'outcome="success",le="0.25"} 1' + ) in rendered + assert ( + 'ai_sre_provider_request_duration_seconds_count{provider="openai",model="approved-model",' + 'outcome="failure"} 1' + ) in rendered + assert ( + 'ai_sre_provider_fallbacks_total{provider="openai",model="approved-model",' + 'reason="provider_request_failed"} 1' + ) in rendered + assert ( + 'ai_sre_provider_tokens_total{provider="openai",model="approved-model",direction="input"} 120' + in rendered + ) + assert ( + 'ai_sre_provider_tokens_total{provider="openai",model="approved-model",direction="output"} 30' + in rendered + ) + + +def test_provider_metrics_keep_label_values_bounded_and_ignore_sensitive_payload_fields(): + metrics = ProviderMetricsRegistry() + + metrics.observe( + { + "provider": "Bearer secret-provider-token", + "model": "model-name", + "attempted": False, + "outcome": "unexpected-outcome", + "request_latency_ms": 123.0, + "fallback_used": True, + "fallback_reason": "arbitrary-error-message", + "prompt": "private incident evidence", + "usage": {"input_tokens": -1, "output_tokens": True}, + } + ) + + rendered = metrics.render_prometheus() + + assert "secret-provider-token" not in rendered + assert "private incident evidence" not in rendered + assert 'outcome="unknown"' in rendered + assert 'reason="other"' in rendered + assert "ai_sre_provider_request_duration_seconds_count" not in rendered + assert "ai_sre_provider_tokens_total{" not in rendered + + +def test_provider_metrics_endpoint_exposes_prometheus_contract(): + response = TestClient(app).get("/metrics") + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/plain") + assert "# TYPE ai_sre_provider_analyses_total counter" in response.text + assert "# TYPE ai_sre_provider_request_duration_seconds histogram" in response.text diff --git a/docs/09-roadmap.md b/docs/09-roadmap.md index 183b9e1..2cc7d36 100644 --- a/docs/09-roadmap.md +++ b/docs/09-roadmap.md @@ -43,13 +43,13 @@ The [commercialization roadmap](21-commercialization-roadmap.md) runs alongside ## Week 5 - Provider Telemetry -- Day 1: expose a bounded per-call contract for provider/model identity, request latency, token usage, outcome, and deterministic fallback without storing sensitive content. -- Add aggregate counters and latency distributions with bounded labels. +- Day 1 - complete: expose a bounded per-call contract for provider/model identity, request latency, token usage, outcome, and deterministic fallback without storing sensitive content. +- Day 2 - complete: add aggregate counters and latency distributions with bounded labels. - Define explicit pricing inputs and estimated cost metadata without hard-coding unstable provider prices. - Join provider usage and cost with evaluation outcomes. - Calculate cost per successful evaluated analysis. -See [Provider Telemetry Contract](22-provider-telemetry.md) for the Day 1 API contract, privacy boundary, and remaining-week sequence. +See [Provider Telemetry Contract](22-provider-telemetry.md) for the Day 1 per-request contract, Day 2 aggregate metrics, privacy boundary, and remaining-week sequence. **Exit gate:** provider usage, reliability, and cost can be compared with evaluation outcomes without storing prompts, incident evidence, credentials, endpoints, or generated content. diff --git a/docs/18-production-observability.md b/docs/18-production-observability.md index e8a2aad..31f62b7 100644 --- a/docs/18-production-observability.md +++ b/docs/18-production-observability.md @@ -64,13 +64,13 @@ A future instrumentation pass can use a bounded metric contract like this: | --- | --- | --- | | `ai_sre_analysis_requests_total` | `endpoint`, `analysis_mode`, `outcome` | Request volume and completion outcomes. | | `ai_sre_analysis_duration_seconds` | `endpoint`, `analysis_mode` | End-to-end latency distribution. | -| `ai_sre_fallbacks_total` | `reason` | Deterministic fallback behavior. | +| `ai_sre_provider_fallbacks_total` | `provider`, `model`, `reason` | Deterministic provider fallback behavior. | | `ai_sre_prompt_truncations_total` | `reason` | Cost-control activation. | | `ai_sre_redactions_total` | `rule_category` | Redaction activity without sensitive values. | | `ai_sre_eval_checks_total` | `suite`, `dimension`, `result`, `assistant_version` | Quality and release-gate trends. | | `ai_sre_provider_tokens_total` | `provider`, `model`, `direction` | Token usage for quality/cost comparisons. | -Week 5 Day 1 implements per-request provider, model, outcome, fallback, latency, and token-usage metadata in API responses. The metric names above remain a proposed aggregate contract. Request IDs and workspace IDs should not be added as metric labels. Per-customer billing or audit attribution belongs in a controlled event ledger with its own access and retention policy. +Week 5 Days 1 and 2 implement per-request metadata plus process-local Prometheus aggregates for provider outcomes, attempted-request latency, deterministic fallbacks, and reported input/output tokens. Broader end-to-end analysis, truncation, redaction, and evaluation metrics above remain proposed. Request IDs and workspace IDs should not be added as metric labels. Per-customer billing or audit attribution belongs in a controlled event ledger with its own access and retention policy. Do not place full prompts, log evidence, model responses, API keys, or incident payloads in metrics. Detailed audit records require separate access, retention, and redaction controls. diff --git a/docs/22-provider-telemetry.md b/docs/22-provider-telemetry.md index b56eda3..326fe15 100644 --- a/docs/22-provider-telemetry.md +++ b/docs/22-provider-telemetry.md @@ -1,6 +1,6 @@ # Provider Telemetry Contract -Week 5, Day 1 establishes the privacy-safe metadata contract for optional LLM calls. +Week 5, Days 1 and 2 establish privacy-safe per-request metadata and process-local aggregate metrics for optional LLM calls. The assistant already bounded prompts and fell back to deterministic analysis. The missing foundation was a consistent way to answer: which provider path was selected, was a request attempted, did it succeed, how long did it take, did the provider report token usage, and did the deterministic fallback protect the user? @@ -29,7 +29,7 @@ When `use_llm=true`, `/ask`, `/analyze/logs`, and `/summarize-incident` include } ``` -The contract is per request. It is not yet a persistent usage ledger, billing record, Prometheus metric implementation, or cost estimate. +This API contract is per request. Day 2 also aggregates its bounded fields as Prometheus text; neither surface is a persistent usage ledger, billing record, or cost estimate. ## Field Semantics @@ -50,6 +50,36 @@ The contract is per request. It is not yet a persistent usage ledger, billing re Unknown, negative, boolean, or malformed token values are treated as unavailable. Providers may omit usage; omission does not turn a successful analysis into a failure. +## Aggregate Metrics + +`GET /metrics` on the AI SRE Assistant exposes four process-local metric families: + +| Metric | Type | Labels | Meaning | +| --- | --- | --- | --- | +| `ai_sre_provider_analyses_total` | Counter | `provider`, `model`, `outcome` | LLM enrichment selections, including successful, failed, and not-configured outcomes. | +| `ai_sre_provider_request_duration_seconds` | Histogram | `provider`, `model`, `outcome` | Latency distribution for attempted provider requests only. | +| `ai_sre_provider_fallbacks_total` | Counter | `provider`, `model`, `reason` | Deterministic fallbacks caused by missing configuration or request failure. | +| `ai_sre_provider_tokens_total` | Counter | `provider`, `model`, `direction` | Provider-reported input and output tokens; missing usage is not treated as zero. | + +The histogram uses fixed buckets from 50 milliseconds through the 30-second request timeout plus `+Inf`. Unconfigured selections increment the analysis and fallback counters but do not create a provider-request latency observation. + +These aggregates are intentionally in memory and reset when the assistant process restarts. Durable metering, billing attribution, retention, multi-process aggregation, and customer-level usage history remain separate production concerns. + +### Label Boundary + +Provider and model values come only from deployment configuration, pass through redaction, and are limited to 100 characters. Outcome, fallback reason, and token direction use fixed enums. The registry maps unexpected enum values to `unknown` or `other` rather than accepting arbitrary strings. + +The aggregate contract never labels metrics with prompts, evidence, generated content, provider endpoints, errors, request IDs, users, workspaces, customers, or incidents. Operators should also keep the number of configured provider/model pairs within an explicit cardinality budget. + +Example: + +```text +ai_sre_provider_analyses_total{provider="openai",model="approved-model",outcome="success"} 12 +ai_sre_provider_request_duration_seconds_bucket{provider="openai",model="approved-model",outcome="success",le="1"} 9 +ai_sre_provider_fallbacks_total{provider="openai",model="approved-model",reason="provider_request_failed"} 2 +ai_sre_provider_tokens_total{provider="openai",model="approved-model",direction="input"} 5040 +``` + ## Outcomes And Fallbacks | Situation | Outcome | Attempted | Fallback | @@ -102,11 +132,21 @@ Do not attach customer identifiers to future Prometheus labels. Per-team billing - Both LLM-enabled API flows expose the same contract. - Success, failure, fallback, usage, latency, and privacy behavior have deterministic tests. +## Day 2 Definition Of Done + +- Provider outcome selections are counted across success, failure, and not-configured paths. +- Attempted requests populate a fixed latency histogram with count and sum. +- Deterministic fallback reasons are counted separately. +- Valid provider-reported input and output tokens are accumulated without converting unknown usage to zero. +- Labels are limited to configured provider/model values and fixed enums. +- Sensitive payload fields and arbitrary error text never enter the aggregate metrics. +- The assistant exposes the contract as Prometheus text at `GET /metrics`. +- Aggregation, histogram, fallback, token, malformed-value, privacy, and endpoint behavior have deterministic tests. + ## Remaining Week 5 Sequence -1. Add bounded aggregate counters and latency distributions without high-cardinality labels. -2. Define explicit model pricing inputs and estimated cost metadata without hard-coding unstable prices. -3. Join provider outcomes and cost with evaluation results to calculate cost per successful evaluated analysis. -4. Add a local comparison report across deterministic and configured provider paths. -5. Exercise provider failure and fallback behavior end to end. -6. Close the week with an exit-gate review against the technical and commercialization roadmaps. +1. Define explicit model pricing inputs and estimated cost metadata without hard-coding unstable prices. +2. Join provider outcomes and cost with evaluation results to calculate cost per successful evaluated analysis. +3. Add a local comparison report across deterministic and configured provider paths. +4. Exercise provider failure and fallback behavior end to end. +5. Close the week with an exit-gate review against the technical and commercialization roadmaps. diff --git a/docs/build-log.md b/docs/build-log.md index d3c0945..a2c2f14 100644 --- a/docs/build-log.md +++ b/docs/build-log.md @@ -714,3 +714,43 @@ What comes next: - Define explicit pricing inputs without hard-coding unstable provider prices. - Join provider usage and cost with evaluation outcomes. - Calculate and report cost per successful evaluated analysis. + +## Week 5, Day 2 - Provider Aggregate Metrics + +Today I turned the per-request provider contract into privacy-safe operational aggregates. + +Day 1 made each provider attempt visible to its caller. Day 2 makes trends scrapeable without retaining prompts, incident evidence, generated content, customer identifiers, or arbitrary error text. + +What changed: + +- Added an in-memory, dependency-free provider metrics registry to the AI SRE Assistant. +- Counted enrichment outcomes across success, failure, and provider-not-configured paths. +- Added a fixed-bucket latency histogram for attempted provider requests. +- Counted deterministic fallbacks by the two supported reasons. +- Accumulated valid provider-reported input and output tokens while keeping missing usage unknown. +- Exposed the aggregates as Prometheus text at `GET /metrics`. +- Limited labels to deployment-configured provider/model values and fixed outcome, reason, and direction enums. +- Mapped unexpected enum values to bounded fallback labels instead of accepting arbitrary strings. +- Added tests for counters, cumulative latency buckets, token totals, fallbacks, malformed values, privacy, and the metrics endpoint. +- Updated the roadmap and telemetry guide to mark Day 2 complete and define the remaining Week 5 sequence. + +Why this matters: + +Per-request metadata helps debug one analysis. Aggregate counters and distributions reveal whether provider enrichment is reliable, fast, and consistently reporting usage across many analyses. That is the evidence needed before pricing and quality are joined later in the week. + +The privacy and cardinality boundaries are part of the feature. Prompts, evidence, errors, endpoints, request IDs, users, workspaces, and incidents never become labels. Process-local metrics also remain honest about their limits: they reset on restart and are not a billing ledger. + +Lessons learned: + +- Latency needs a distribution; an average hides slow tails. +- Not configured is an outcome, but it is not an attempted provider request and should not enter request-latency buckets. +- Missing token usage is unknown, not zero. +- Configured provider/model labels are useful only with an explicit cardinality budget. +- Operational metrics and durable customer metering are different systems with different access and retention requirements. + +What comes next: + +- Define explicit pricing inputs without hard-coding unstable provider prices. +- Calculate estimated cost metadata from reported usage. +- Join cost and provider outcomes with evaluation results. +- Report cost per successful evaluated analysis.