diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 22f3592..ea05b80 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing -Thanks for helping improve `ai-infra-starter-kit`. +Thanks for helping improve AIOps Lab. The main rule is simple: keep the project beginner-friendly without making it shallow. A new feature should teach a real AI infrastructure concept at the right time. diff --git a/LICENSE b/LICENSE index cfa6ae6..1fcc42a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2026 ai-infra-starter-kit contributors +Copyright (c) 2026 AIOps Lab contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 48a957e..4198e91 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Simple first. Production-minded always. -`ai-infra-starter-kit` is a practical learning lab for AI infrastructure. It starts with a normal web service, adds logs and metrics, then uses an AI SRE Assistant to explain what is happening operationally. +AIOps Lab is a practical learning lab for AI infrastructure. It starts with a normal web service, adds logs and metrics, then uses an AI SRE Assistant to explain what is happening operationally. The first version runs on a normal laptop. No GPU, Kubernetes, vLLM, Triton, Ray, KServe, or full MLOps platform is required on Day 1. @@ -41,8 +41,8 @@ The AI assistant does not need an LLM key to work. If no provider is configured, ## Quickstart ```bash -git clone https://github.com/your-username/ai-infra-starter-kit.git -cd ai-infra-starter-kit +git clone https://github.com/utkarshp845/AIOps-Lab.git +cd AIOps-Lab cp .env.example .env make up make test @@ -97,6 +97,8 @@ For the advanced model serving decision framework and commercialization path, se For the Week 4 release gates, readiness verdict, and next-stage priorities, see `docs/20-production-readiness-review.md`. +For the phased commercialization roadmap with milestones and success metrics, see `docs/21-commercialization-roadmap.md`. + Ask the assistant directly: ```bash @@ -124,7 +126,7 @@ curl -s -X POST http://localhost:8001/summarize-incident \ ## Repository Structure ```text -ai-infra-starter-kit/ +AIOps-Lab/ apps/ demo-service/ # FastAPI app that emits health, failure, latency, logs, and metrics ai-sre-assistant/ # FastAPI app and CLI that analyze demo-service logs @@ -141,6 +143,7 @@ ai-infra-starter-kit/ 18-production-observability.md 19-advanced-model-serving-roadmap.md 20-production-readiness-review.md + 21-commercialization-roadmap.md infra/ # Docker, Kubernetes, and Terraform starter notes k8s/ # kind-first Kubernetes manifests and walkthrough scripts/ # Local traffic and log helper scripts @@ -190,6 +193,8 @@ The open-source project makes the core learning path, safety controls, and evalu The future paid opportunity is the recurring team workflow around those foundations: hosted evaluation history, private incident datasets, release gates, model and provider comparisons, regression alerts, collaboration, and audit-ready exports. The product value is not simply generating an incident summary. It is helping teams prove that their operational assistant stays useful, safe, private, and cost-aware as their systems change. +The sequenced plan — audience first, wedge product with design partners second, paid team tier third — lives in `docs/21-commercialization-roadmap.md` with per-phase milestones, success metrics, and exit criteria. + ## Lessons Learned Day 1 lessons are intentionally simple: diff --git a/SECURITY.md b/SECURITY.md index a27409f..2cccec7 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,6 +1,6 @@ # Security Policy -`ai-infra-starter-kit` is a learning project, but security issues still matter. +AIOps Lab is a learning project, but security issues still matter. This repo intentionally starts local and simple. Please do not treat the default Docker Compose or kind setup as production-ready. diff --git a/docker-compose.yml b/docker-compose.yml index c64ebb8..c01e765 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,10 @@ -name: ai-infra-starter-kit +name: aiops-lab services: demo-service: build: context: ./apps/demo-service - container_name: ai-infra-demo-service + container_name: aiops-demo-service ports: - "8000:8000" environment: @@ -20,7 +20,7 @@ services: ai-sre-assistant: build: context: ./apps/ai-sre-assistant - container_name: ai-infra-ai-sre-assistant + container_name: aiops-ai-sre-assistant ports: - "8001:8001" environment: @@ -46,11 +46,11 @@ services: # Optional local model provider for later experiments. # 1. Uncomment this service. # 2. Set LLM_PROVIDER=ollama and OPENAI_BASE_URL=http://ollama:11434/v1 in .env. - # 3. Pull a model with: docker exec -it ai-infra-ollama ollama pull llama3.1 + # 3. Pull a model with: docker exec -it aiops-ollama ollama pull llama3.1 # # ollama: # image: ollama/ollama:latest - # container_name: ai-infra-ollama + # container_name: aiops-ollama # ports: # - "11434:11434" # volumes: diff --git a/docs/00-overview.md b/docs/00-overview.md index 5938ccd..46efc81 100644 --- a/docs/00-overview.md +++ b/docs/00-overview.md @@ -1,6 +1,6 @@ # Overview -`ai-infra-starter-kit` teaches AI infrastructure by starting with the parts every production engineer already understands: services, logs, health checks, metrics, containers, and operational debugging. +AIOps Lab teaches AI infrastructure by starting with the parts every production engineer already understands: services, logs, health checks, metrics, containers, and operational debugging. The AI part is introduced as an assistant that observes a service, not as a giant platform on Day 1. diff --git a/docs/09-roadmap.md b/docs/09-roadmap.md index 1714609..f7e11bd 100644 --- a/docs/09-roadmap.md +++ b/docs/09-roadmap.md @@ -35,6 +35,10 @@ - Optional advanced serving roadmap: vLLM, Triton, Ray, KServe, and GPU scheduling. - Production-readiness review with local and CI release gates. +## Commercialization + +The phased plan for turning the lab into a monetized product — audience, wedge product, paid team tier — is in `21-commercialization-roadmap.md`. + ## Later - OpenTelemetry collector. diff --git a/docs/10-kubernetes-operations-runbook.md b/docs/10-kubernetes-operations-runbook.md index 7cbe8bc..1c7ed35 100644 --- a/docs/10-kubernetes-operations-runbook.md +++ b/docs/10-kubernetes-operations-runbook.md @@ -17,23 +17,23 @@ This runbook assumes you already created the kind cluster, loaded local images, All Kubernetes resources for this project run in one namespace: ```bash -kubectl get all -n ai-infra-starter-kit +kubectl get all -n aiops-lab ``` For a shorter local session, you can set the namespace on your current context: ```bash -kubectl config set-context --current --namespace=ai-infra-starter-kit +kubectl config set-context --current --namespace=aiops-lab ``` -The rest of this guide keeps `-n ai-infra-starter-kit` in commands so each example is explicit and copy-paste friendly. +The rest of this guide keeps `-n aiops-lab` in commands so each example is explicit and copy-paste friendly. ## Quick Status Check Start with the cluster objects that explain most local issues: ```bash -kubectl get deployments,pods,svc,pvc -n ai-infra-starter-kit +kubectl get deployments,pods,svc,pvc -n aiops-lab ``` What you want to see: @@ -47,8 +47,8 @@ What you want to see: Then confirm rollout status: ```bash -kubectl rollout status deployment/demo-service -n ai-infra-starter-kit -kubectl rollout status deployment/ai-sre-assistant -n ai-infra-starter-kit +kubectl rollout status deployment/demo-service -n aiops-lab +kubectl rollout status deployment/ai-sre-assistant -n aiops-lab ``` ## Health, Readiness, And Metrics @@ -56,11 +56,11 @@ kubectl rollout status deployment/ai-sre-assistant -n ai-infra-starter-kit Port-forward both services in separate terminals: ```bash -kubectl port-forward svc/demo-service 8000:8000 -n ai-infra-starter-kit +kubectl port-forward svc/demo-service 8000:8000 -n aiops-lab ``` ```bash -kubectl port-forward svc/ai-sre-assistant 8001:8001 -n ai-infra-starter-kit +kubectl port-forward svc/ai-sre-assistant 8001:8001 -n aiops-lab ``` Check the app-level signals: @@ -117,14 +117,14 @@ For an end-to-end incident walkthrough that uses these signals together, see `13 View recent logs from each Deployment: ```bash -kubectl logs deployment/demo-service -n ai-infra-starter-kit --tail=50 -kubectl logs deployment/ai-sre-assistant -n ai-infra-starter-kit --tail=50 +kubectl logs deployment/demo-service -n aiops-lab --tail=50 +kubectl logs deployment/ai-sre-assistant -n aiops-lab --tail=50 ``` Follow logs while generating traffic: ```bash -kubectl logs deployment/demo-service -n ai-infra-starter-kit --follow +kubectl logs deployment/demo-service -n aiops-lab --follow ``` For this learning lab, the assistant also reads the shared file at: @@ -142,14 +142,14 @@ Services give pods stable names inside the cluster. Check service objects: ```bash -kubectl get svc -n ai-infra-starter-kit +kubectl get svc -n aiops-lab ``` Check service endpoints: ```bash -kubectl get endpoints demo-service -n ai-infra-starter-kit -kubectl get endpoints ai-sre-assistant -n ai-infra-starter-kit +kubectl get endpoints demo-service -n aiops-lab +kubectl get endpoints ai-sre-assistant -n aiops-lab ``` If a Service has no endpoints, Kubernetes has not found any ready pods behind it. Check pod labels, readiness probes, and pod status. @@ -159,13 +159,13 @@ If a Service has no endpoints, Kubernetes has not found any ready pods behind it Normal configuration lives in the ConfigMap. For the full config and secrets walkthrough, see `11-kubernetes-config-and-secrets.md`: ```bash -kubectl describe configmap ai-infra-starter-kit-config -n ai-infra-starter-kit +kubectl describe configmap aiops-lab-config -n aiops-lab ``` Sensitive configuration belongs in a Secret: ```bash -kubectl get secret ai-sre-assistant-secrets -n ai-infra-starter-kit +kubectl get secret ai-sre-assistant-secrets -n aiops-lab ``` Do not paste real API keys into screenshots, tweets, issues, or pull requests. @@ -177,13 +177,13 @@ The local kind setup uses a shared PersistentVolumeClaim so the assistant can re Check the claim: ```bash -kubectl get pvc -n ai-infra-starter-kit +kubectl get pvc -n aiops-lab ``` Describe it if it is not `Bound`: ```bash -kubectl describe pvc shared-logs -n ai-infra-starter-kit +kubectl describe pvc shared-logs -n aiops-lab ``` This is a learning bridge from Docker Compose to Kubernetes. It is not the recommended production logging architecture. @@ -197,18 +197,18 @@ Likely cause: kind cannot see the local Docker image. Check: ```bash -kubectl describe pod -l app.kubernetes.io/name=demo-service -n ai-infra-starter-kit +kubectl describe pod -l app.kubernetes.io/name=demo-service -n aiops-lab ``` Fix: ```bash -docker build -t ai-infra-starter-kit/demo-service:local apps/demo-service -docker build -t ai-infra-starter-kit/ai-sre-assistant:local apps/ai-sre-assistant -kind load docker-image ai-infra-starter-kit/demo-service:local --name ai-infra-starter-kit -kind load docker-image ai-infra-starter-kit/ai-sre-assistant:local --name ai-infra-starter-kit -kubectl rollout restart deployment/demo-service -n ai-infra-starter-kit -kubectl rollout restart deployment/ai-sre-assistant -n ai-infra-starter-kit +docker build -t aiops-lab/demo-service:local apps/demo-service +docker build -t aiops-lab/ai-sre-assistant:local apps/ai-sre-assistant +kind load docker-image aiops-lab/demo-service:local --name aiops-lab +kind load docker-image aiops-lab/ai-sre-assistant:local --name aiops-lab +kubectl rollout restart deployment/demo-service -n aiops-lab +kubectl rollout restart deployment/ai-sre-assistant -n aiops-lab ``` ### Pods Are Running But Not Ready @@ -218,8 +218,8 @@ Likely cause: readiness probes are failing or the app is not listening on the ex Check: ```bash -kubectl describe pod -l app.kubernetes.io/name=demo-service -n ai-infra-starter-kit -kubectl logs deployment/demo-service -n ai-infra-starter-kit --tail=100 +kubectl describe pod -l app.kubernetes.io/name=demo-service -n aiops-lab +kubectl logs deployment/demo-service -n aiops-lab --tail=100 ``` Look for failed readiness probe events, startup errors, or port mismatches. @@ -236,10 +236,10 @@ Likely causes: Check: ```bash -kubectl get pvc -n ai-infra-starter-kit -kubectl describe configmap ai-infra-starter-kit-config -n ai-infra-starter-kit -kubectl logs deployment/demo-service -n ai-infra-starter-kit --tail=50 -kubectl logs deployment/ai-sre-assistant -n ai-infra-starter-kit --tail=50 +kubectl get pvc -n aiops-lab +kubectl describe configmap aiops-lab-config -n aiops-lab +kubectl logs deployment/demo-service -n aiops-lab --tail=50 +kubectl logs deployment/ai-sre-assistant -n aiops-lab --tail=50 ``` Then generate traffic again: @@ -259,8 +259,8 @@ Likely causes: Check: ```bash -kubectl get endpoints demo-service -n ai-infra-starter-kit -kubectl describe configmap ai-infra-starter-kit-config -n ai-infra-starter-kit +kubectl get endpoints demo-service -n aiops-lab +kubectl describe configmap aiops-lab-config -n aiops-lab curl http://localhost:8000/metrics ``` @@ -278,7 +278,7 @@ Check: ```bash kubectl get pv -kubectl describe pvc shared-logs -n ai-infra-starter-kit +kubectl describe pvc shared-logs -n aiops-lab ``` For this repo, `infra/k8s/storage.yaml` includes a kind-friendly local PersistentVolume. Reapply it if needed: @@ -298,14 +298,14 @@ Likely causes: Check: ```bash -kubectl get svc -n ai-infra-starter-kit +kubectl get svc -n aiops-lab ``` Use different local ports if needed: ```bash -kubectl port-forward svc/demo-service 18000:8000 -n ai-infra-starter-kit -kubectl port-forward svc/ai-sre-assistant 18001:8001 -n ai-infra-starter-kit +kubectl port-forward svc/demo-service 18000:8000 -n aiops-lab +kubectl port-forward svc/ai-sre-assistant 18001:8001 -n aiops-lab ``` Then call: @@ -319,7 +319,7 @@ curl http://localhost:18001/health Use this order when something is wrong: -1. `kubectl get deployments,pods,svc,pvc -n ai-infra-starter-kit` +1. `kubectl get deployments,pods,svc,pvc -n aiops-lab` 2. `kubectl describe pod ...` 3. `kubectl logs deployment/...` 4. app health endpoints through port-forward @@ -333,8 +333,8 @@ Start with read-only inspection. Restart or delete resources only after the evid Useful screenshots for a build-in-public update: -- `kubectl get deployments,pods,svc,pvc -n ai-infra-starter-kit` -- `kubectl rollout status deployment/demo-service -n ai-infra-starter-kit` +- `kubectl get deployments,pods,svc,pvc -n aiops-lab` +- `kubectl rollout status deployment/demo-service -n aiops-lab` - `curl http://localhost:8000/health` and `curl http://localhost:8000/metrics` - `POST /analyze/metrics` response from the assistant - `kubectl logs deployment/demo-service --tail=20` diff --git a/docs/11-kubernetes-config-and-secrets.md b/docs/11-kubernetes-config-and-secrets.md index 5caadac..2510b42 100644 --- a/docs/11-kubernetes-config-and-secrets.md +++ b/docs/11-kubernetes-config-and-secrets.md @@ -40,7 +40,7 @@ Current values: Inspect the ConfigMap: ```bash -kubectl describe configmap ai-infra-starter-kit-config -n ai-infra-starter-kit +kubectl describe configmap aiops-lab-config -n aiops-lab ``` The default `LLM_PROVIDER` is `none` so the project works without an API key. The default cost controls keep provider prompts bounded when a user enables LLM enrichment. @@ -60,7 +60,7 @@ The assistant deployment marks this secret key as optional. That means Kubernete Inspect whether the Secret exists: ```bash -kubectl get secret ai-sre-assistant-secrets -n ai-infra-starter-kit +kubectl get secret ai-sre-assistant-secrets -n aiops-lab ``` Do not print, screenshot, tweet, or commit real secret values. @@ -116,7 +116,7 @@ Create a private local Secret manifest: ```bash kubectl create secret generic ai-sre-assistant-secrets \ --from-literal=OPENAI_API_KEY="$OPENAI_API_KEY" \ - -n ai-infra-starter-kit \ + -n aiops-lab \ --dry-run=client \ -o yaml > infra/k8s/secret.local.yaml ``` @@ -126,7 +126,7 @@ PowerShell: ```powershell kubectl create secret generic ai-sre-assistant-secrets ` --from-literal=OPENAI_API_KEY="$env:OPENAI_API_KEY" ` - -n ai-infra-starter-kit ` + -n aiops-lab ` --dry-run=client ` -o yaml > infra/k8s/secret.local.yaml ``` @@ -140,8 +140,8 @@ kubectl apply -f infra/k8s/secret.local.yaml Enable the provider in the ConfigMap: ```bash -kubectl patch configmap ai-infra-starter-kit-config \ - -n ai-infra-starter-kit \ +kubectl patch configmap aiops-lab-config \ + -n aiops-lab \ --type merge \ -p '{"data":{"LLM_PROVIDER":"openai"}}' ``` @@ -149,8 +149,8 @@ kubectl patch configmap ai-infra-starter-kit-config \ Restart the assistant so it reads the updated environment variables: ```bash -kubectl rollout restart deployment/ai-sre-assistant -n ai-infra-starter-kit -kubectl rollout status deployment/ai-sre-assistant -n ai-infra-starter-kit +kubectl rollout restart deployment/ai-sre-assistant -n aiops-lab +kubectl rollout status deployment/ai-sre-assistant -n aiops-lab ``` Why restart? Environment variables from ConfigMaps and Secrets are read when the container starts. Updating the ConfigMap or Secret does not automatically update an already-running process. @@ -174,7 +174,7 @@ env: - name: LLM_PROVIDER valueFrom: configMapKeyRef: - name: ai-infra-starter-kit-config + name: aiops-lab-config key: LLM_PROVIDER - name: OPENAI_API_KEY valueFrom: @@ -193,7 +193,7 @@ This makes the app image reusable. The image does not need to be rebuilt when co Check the provider value: ```bash -kubectl describe configmap ai-infra-starter-kit-config -n ai-infra-starter-kit +kubectl describe configmap aiops-lab-config -n aiops-lab ``` If `LLM_PROVIDER=none`, the assistant is behaving as configured. @@ -201,13 +201,13 @@ If `LLM_PROVIDER=none`, the assistant is behaving as configured. Check whether the Secret exists: ```bash -kubectl get secret ai-sre-assistant-secrets -n ai-infra-starter-kit +kubectl get secret ai-sre-assistant-secrets -n aiops-lab ``` Restart the Deployment after changing ConfigMaps or Secrets: ```bash -kubectl rollout restart deployment/ai-sre-assistant -n ai-infra-starter-kit +kubectl rollout restart deployment/ai-sre-assistant -n aiops-lab ``` ### The Secret Exists But The LLM Request Fails @@ -215,7 +215,7 @@ kubectl rollout restart deployment/ai-sre-assistant -n ai-infra-starter-kit Check assistant logs without printing the key: ```bash -kubectl logs deployment/ai-sre-assistant -n ai-infra-starter-kit --tail=100 +kubectl logs deployment/ai-sre-assistant -n aiops-lab --tail=100 ``` Likely causes: @@ -231,9 +231,9 @@ Likely causes: Delete and recreate the Secret: ```bash -kubectl delete secret ai-sre-assistant-secrets -n ai-infra-starter-kit +kubectl delete secret ai-sre-assistant-secrets -n aiops-lab kubectl apply -f infra/k8s/secret.local.yaml -kubectl rollout restart deployment/ai-sre-assistant -n ai-infra-starter-kit +kubectl rollout restart deployment/ai-sre-assistant -n aiops-lab ``` Do not commit `infra/k8s/secret.local.yaml`. diff --git a/docs/12-kubernetes-health-and-resources.md b/docs/12-kubernetes-health-and-resources.md index 9dc1880..053b250 100644 --- a/docs/12-kubernetes-health-and-resources.md +++ b/docs/12-kubernetes-health-and-resources.md @@ -125,33 +125,33 @@ Useful conversions: Describe the Deployment: ```bash -kubectl describe deployment demo-service -n ai-infra-starter-kit -kubectl describe deployment ai-sre-assistant -n ai-infra-starter-kit +kubectl describe deployment demo-service -n aiops-lab +kubectl describe deployment ai-sre-assistant -n aiops-lab ``` Describe a pod: ```bash -kubectl describe pod -l app.kubernetes.io/name=demo-service -n ai-infra-starter-kit +kubectl describe pod -l app.kubernetes.io/name=demo-service -n aiops-lab ``` Check events: ```bash -kubectl get events -n ai-infra-starter-kit --sort-by=.lastTimestamp +kubectl get events -n aiops-lab --sort-by=.lastTimestamp ``` Check pod status: ```bash -kubectl get pods -n ai-infra-starter-kit +kubectl get pods -n aiops-lab ``` Check Service endpoints: ```bash -kubectl get endpoints demo-service -n ai-infra-starter-kit -kubectl get endpoints ai-sre-assistant -n ai-infra-starter-kit +kubectl get endpoints demo-service -n aiops-lab +kubectl get endpoints ai-sre-assistant -n aiops-lab ``` If a pod is running but not ready, it may not appear as a ready endpoint behind the Service. @@ -161,7 +161,7 @@ If a pod is running but not ready, it may not appear as a ready endpoint behind Port-forward the demo service: ```bash -kubectl port-forward svc/demo-service 8000:8000 -n ai-infra-starter-kit +kubectl port-forward svc/demo-service 8000:8000 -n aiops-lab ``` Check health and readiness: @@ -174,7 +174,7 @@ curl http://localhost:8000/ready Port-forward the assistant: ```bash -kubectl port-forward svc/ai-sre-assistant 8001:8001 -n ai-infra-starter-kit +kubectl port-forward svc/ai-sre-assistant 8001:8001 -n aiops-lab ``` Check assistant health: @@ -188,19 +188,19 @@ curl http://localhost:8001/health Restart a Deployment: ```bash -kubectl rollout restart deployment/demo-service -n ai-infra-starter-kit +kubectl rollout restart deployment/demo-service -n aiops-lab ``` Watch rollout status: ```bash -kubectl rollout status deployment/demo-service -n ai-infra-starter-kit +kubectl rollout status deployment/demo-service -n aiops-lab ``` Watch pods: ```bash -kubectl get pods -n ai-infra-starter-kit --watch +kubectl get pods -n aiops-lab --watch ``` This is a useful way to see readiness in action. Kubernetes starts the new pod, waits for it to become ready, and then the Deployment becomes available. @@ -212,7 +212,7 @@ This is a useful way to see readiness in action. Kubernetes starts the new pod, Check events and probe failures: ```bash -kubectl describe pod -l app.kubernetes.io/name=demo-service -n ai-infra-starter-kit +kubectl describe pod -l app.kubernetes.io/name=demo-service -n aiops-lab ``` Then check the app endpoint directly through port-forward: @@ -233,19 +233,19 @@ Likely causes: Check restart count: ```bash -kubectl get pods -n ai-infra-starter-kit +kubectl get pods -n aiops-lab ``` Describe the pod: ```bash -kubectl describe pod -l app.kubernetes.io/name=demo-service -n ai-infra-starter-kit +kubectl describe pod -l app.kubernetes.io/name=demo-service -n aiops-lab ``` Check logs: ```bash -kubectl logs deployment/demo-service -n ai-infra-starter-kit --tail=100 +kubectl logs deployment/demo-service -n aiops-lab --tail=100 ``` Likely causes: @@ -262,7 +262,7 @@ Likely causes: Check pod details: ```bash -kubectl describe pod -l app.kubernetes.io/name=demo-service -n ai-infra-starter-kit +kubectl describe pod -l app.kubernetes.io/name=demo-service -n aiops-lab ``` In a real system, next steps might include: diff --git a/docs/13-kubernetes-incident-debugging.md b/docs/13-kubernetes-incident-debugging.md index 3eac4f4..97cf22d 100644 --- a/docs/13-kubernetes-incident-debugging.md +++ b/docs/13-kubernetes-incident-debugging.md @@ -37,11 +37,11 @@ You should have: Port-forward commands: ```bash -kubectl port-forward svc/demo-service 8000:8000 -n ai-infra-starter-kit +kubectl port-forward svc/demo-service 8000:8000 -n aiops-lab ``` ```bash -kubectl port-forward svc/ai-sre-assistant 8001:8001 -n ai-infra-starter-kit +kubectl port-forward svc/ai-sre-assistant 8001:8001 -n aiops-lab ``` ## Step 1: Create Symptoms @@ -83,13 +83,13 @@ This creates enough evidence for Kubernetes inspection, metrics analysis, and as Start broad: ```bash -kubectl get deployments,pods,svc,endpoints,pvc -n ai-infra-starter-kit +kubectl get deployments,pods,svc,endpoints,pvc -n aiops-lab ``` Then check recent events: ```bash -kubectl get events -n ai-infra-starter-kit --sort-by=.lastTimestamp +kubectl get events -n aiops-lab --sort-by=.lastTimestamp ``` What this tells you: @@ -125,13 +125,13 @@ If `/health` and `/ready` are passing while errors still happen, that usually me Read recent `demo-service` container logs: ```bash -kubectl logs deployment/demo-service -n ai-infra-starter-kit --tail=100 +kubectl logs deployment/demo-service -n aiops-lab --tail=100 ``` Read assistant logs: ```bash -kubectl logs deployment/ai-sre-assistant -n ai-infra-starter-kit --tail=100 +kubectl logs deployment/ai-sre-assistant -n aiops-lab --tail=100 ``` Look for evidence such as: @@ -230,9 +230,9 @@ The right next step depends on which layer has evidence. Prefer read-only commands first: ```bash -kubectl get deployments,pods,svc,endpoints,pvc -n ai-infra-starter-kit -kubectl get events -n ai-infra-starter-kit --sort-by=.lastTimestamp -kubectl logs deployment/demo-service -n ai-infra-starter-kit --tail=100 +kubectl get deployments,pods,svc,endpoints,pvc -n aiops-lab +kubectl get events -n aiops-lab --sort-by=.lastTimestamp +kubectl logs deployment/demo-service -n aiops-lab --tail=100 curl http://localhost:8000/metrics curl -s -X POST http://localhost:8001/summarize-incident -H "Content-Type: application/json" -d '{"max_lines":200,"use_llm":false}' ``` diff --git a/docs/14-kubernetes-production-next-steps.md b/docs/14-kubernetes-production-next-steps.md index a88e8aa..a17d7f2 100644 --- a/docs/14-kubernetes-production-next-steps.md +++ b/docs/14-kubernetes-production-next-steps.md @@ -1,6 +1,6 @@ # Kubernetes Production Next Steps -Week 3 Day 7 closes the first Kubernetes chapter of `ai-infra-starter-kit`. +Week 3 Day 7 closes the first Kubernetes chapter of AIOps Lab. The local kind setup now teaches the core mapping: diff --git a/docs/21-commercialization-roadmap.md b/docs/21-commercialization-roadmap.md new file mode 100644 index 0000000..4ab6a60 --- /dev/null +++ b/docs/21-commercialization-roadmap.md @@ -0,0 +1,123 @@ +# Commercialization Roadmap + +This document turns the product direction from `docs/19-advanced-model-serving-roadmap.md` and the next-stage priorities from `docs/20-production-readiness-review.md` into a sequenced plan with milestones and success metrics. + +The strategy is deliberate: audience first, wedge product second, paid tiers third. Each phase funds and de-risks the next, and no phase starts before the previous phase's exit criteria are met. + +## Strategy Summary + +The AI incident-copilot market is crowded with well-funded products. AIOps Lab does not win by out-building them from a standing start. It wins by: + +1. Teaching an underserved audience (production engineers new to AI infra) and earning distribution through the open learning lab. +2. Converting that audience into customer discovery for a narrow wedge: an evidence-grounded incident analyst whose quality is enforced by a CI evaluation gate. +3. Monetizing the trust and governance layer — evaluation history, private incident sets, redaction policy, audit — not the inference engine. + +```mermaid +flowchart LR + audience["Phase A
audience and learning lab"] --> wedge["Phase B
wedge product with design partners"] + wedge --> paid["Phase C
paid team tier"] + audience -. "customer discovery" .-> wedge + wedge -. "usage evidence" .-> paid +``` + +## Phase A: Public Release and Audience (Months 0-3) + +Goal: publish the learning lab, build a distribution channel, and validate that the audience exists and will engage. + +### Milestones + +| Milestone | Definition of done | +| --- | --- | +| Public release | `make validate` passes, naming is consistent, and the repository is public with a clear README, license, and contribution guide. | +| Launch posts | A "Show HN" or equivalent launch post plus at least one written walkthrough of an incident analysis session. | +| Content cadence | One build-in-public post per week for eight consecutive weeks, sourced from `docs/build-log.md` and the incident walkthroughs. | +| Contact channel | An email list or equivalent owned channel that readers can join from the README. | +| First paid validation | One small paid artifact live: a deep-dive course, ebook, workshop, or sponsorship. Revenue target is trivial; the goal is proof that someone pays. | +| Customer discovery | Fifteen structured conversations with SRE, DevOps, or platform engineers who used or starred the project, each answering: "What would it take for you to point this at your real logs?" | + +### Success metrics + +- 1,000+ GitHub stars or 200+ unique cloners per month. +- 300+ owned-channel subscribers. +- 15 completed discovery interviews with written notes. +- At least 3 interviewees who describe a current, painful incident-analysis workflow they would change. +- First dollar of revenue from any source. + +### Exit criteria for Phase B + +Proceed only when discovery interviews converge on a repeated, specific pain and at least three teams volunteer to try the assistant against their own sanitized telemetry. If interviews do not converge, stay in Phase A and adjust the audience or the message — do not start building the wedge on guesses. + +## Phase B: Wedge Product with Design Partners (Months 3-6) + +Goal: make the assistant useful against real telemetry for two or three design-partner teams, free, in exchange for feedback and permission to reference them. + +The wedge thesis: teams will adopt an incident analyst they can hold to a measurable quality bar. The evaluation harness is the differentiator; the assistant is the demo. + +### Milestones + +| Milestone | Definition of done | +| --- | --- | +| First real connector | Read-only connector to one real log source (CloudWatch Logs, Loki, or Datadog), honoring the existing bounded-query and redaction policies. The shared-file path remains the learning default. | +| Provider metering | Model identity, latency, token usage, fallback outcome, and cost per successful analysis captured as metrics without storing prompt content (item 1 from `docs/20-production-readiness-review.md`). | +| Private incident sets | A team can record sanitized incidents from their own systems and run the evaluation corpus against them in CI, blocking regressions. | +| Minimum viable trust | OIDC/SSO authentication, per-user audit log of evidence access, and a documented single-tenant deployment using the existing Kubernetes manifests. | +| Design partners | Two or three teams running the assistant against sanitized real data weekly, with a shared feedback channel and written usage notes. | + +### Success metrics + +- 2-3 active design partners with weekly usage (not one-time trials). +- 10+ sanitized real incidents in partner evaluation sets. +- Assistant passes the five-dimension rubric (grounded, useful, safe, private, honest) on partner incidents, not just the public corpus. +- Measured cost per successful analysis for each provider configuration. +- At least one partner quote or case study approved for public use. +- Zero privacy or safety gate failures on partner data. Any failure stops the phase until resolved. + +### Exit criteria for Phase C + +Proceed when at least two design partners say they would pay to keep the workflow, and can articulate the budget it would come from. "This is neat" is not an exit signal; "we would lose something we now rely on" is. + +## Phase C: Paid Team Tier (Months 6-12) + +Goal: convert the design-partner workflow into the Team tier described in `docs/19-advanced-model-serving-roadmap.md`, with pricing validated against real willingness to pay. + +### Milestones + +| Milestone | Definition of done | +| --- | --- | +| Hosted evaluation history | Evaluation runs, thresholds, and pass/fail history stored per team with regression alerts, versioned alongside model, prompt, and corpus (the versioning contract from `docs/20-production-readiness-review.md`). | +| Usage and cost dashboards | Per-team visibility into analyses run, provider spend, fallback rate, and quality trend. | +| Quotas and budgets | Per-team token and spend limits with graceful degradation to the deterministic path. | +| Pricing and packaging | Public pricing page for the Team tier (per-seat) and a contact path for single-tenant private deployment (annual contract). Prices set from design-partner conversations, not guesses. | +| Billing and terms | Payment, invoicing, terms of service, privacy policy, and a support commitment appropriate to the price point. | +| First paying customers | Three paying teams, at least one converted from a design partner. | + +### Success metrics + +- 3+ paying teams; $1,000+ monthly recurring revenue as a floor signal, with growth month over month. +- Net revenue retention: no paying team churns within the first two quarters. +- Cost per successful analysis is measured, published internally, and gross-margin positive at the chosen price. +- Support load is sustainable for a single maintainer (or a hiring/contracting decision is made explicitly). +- Community tier remains fully useful: deterministic analysis, provider configuration, public corpus, and deployment guidance stay open. + +## Explicit Non-Goals Until Gated + +These stay behind the adoption gates in `docs/19-advanced-model-serving-roadmap.md` and require a named customer requirement: + +- GPU deployment examples, vLLM, Triton, Ray Serve, KServe. +- Multi-tenant shared infrastructure (single-tenant deployments come first). +- A general observability platform. The product analyzes evidence; it does not replace Datadog, Grafana, or a log backend. +- Enterprise features (RBAC, policy controls, regional data handling) before an enterprise buyer exists. +- Fundraising decisions. This roadmap assumes bootstrap economics; revisit only with Phase C evidence. + +## Operating Cadence + +- **Weekly:** one build-in-public post; review metrics for the current phase. +- **Monthly:** compare actuals to the current phase's success metrics; write a one-paragraph verdict in `docs/build-log.md`. +- **Per phase:** hold the exit-criteria review before starting the next phase. Skipping a gate requires writing down why, what evidence overrides it, and what would trigger a rollback to the prior phase. + +## Risks and Honest Caveats + +- **Crowded market:** incumbents can ship an adequate incident summary as a feature. The defense is the evaluation-gated trust workflow and the audience, not the summary itself. +- **Audience does not convert:** educational audiences often read without buying. The Phase A paid artifact exists to test conversion early and cheaply. +- **Single maintainer:** every phase must remain operable by one person; any milestone that is not gets cut or simplified. +- **Privacy is existential:** one leaked secret in an analysis output would end trust in the product. Privacy and safety evaluation gates remain hard no-go gates in every phase, exactly as `docs/20-production-readiness-review.md` defines them. diff --git a/docs/build-log.md b/docs/build-log.md index 0a2a0ea..959dde5 100644 --- a/docs/build-log.md +++ b/docs/build-log.md @@ -2,7 +2,7 @@ ## Day 1 - Simple First, Production-Minded Always -I am building `ai-infra-starter-kit` because AI infrastructure is becoming part of normal software infrastructure, but the learning path often starts too late in the stack. +I am building AIOps Lab because AI infrastructure is becoming part of normal software infrastructure, but the learning path often starts too late in the stack. AI infra is intimidating because beginners are quickly surrounded by model serving frameworks, GPU scheduling, Kubernetes operators, distributed systems, observability, evals, and cost tradeoffs. Those topics matter, but they do not need to arrive all at once. diff --git a/infra/k8s/README.md b/infra/k8s/README.md index 6c55456..84327ee 100644 --- a/infra/k8s/README.md +++ b/infra/k8s/README.md @@ -18,7 +18,7 @@ This keeps the Week 3 path close to the existing Docker workflow. ## What Each File Does -- `namespace.yaml`: creates the `ai-infra-starter-kit` namespace. +- `namespace.yaml`: creates the `aiops-lab` namespace. - `configmap.yaml`: stores non-secret environment variables. - `secret.example.yaml`: documents the optional `OPENAI_API_KEY` Secret shape. Do not put real keys in this file. - `storage.yaml`: creates local shared storage for demo logs. @@ -57,13 +57,13 @@ kubectl version --client ## 1. Create A Local Cluster ```bash -kind create cluster --name ai-infra-starter-kit +kind create cluster --name aiops-lab ``` Check the cluster: ```bash -kubectl cluster-info --context kind-ai-infra-starter-kit +kubectl cluster-info --context kind-aiops-lab ``` ## 2. Build Local Images @@ -71,8 +71,8 @@ kubectl cluster-info --context kind-ai-infra-starter-kit From the repository root: ```bash -docker build -t ai-infra-starter-kit/demo-service:local apps/demo-service -docker build -t ai-infra-starter-kit/ai-sre-assistant:local apps/ai-sre-assistant +docker build -t aiops-lab/demo-service:local apps/demo-service +docker build -t aiops-lab/ai-sre-assistant:local apps/ai-sre-assistant ``` ## 3. Load Images Into kind @@ -80,8 +80,8 @@ docker build -t ai-infra-starter-kit/ai-sre-assistant:local apps/ai-sre-assistan kind runs its own container runtime inside the cluster node. Loading the images makes them available to Kubernetes. ```bash -kind load docker-image ai-infra-starter-kit/demo-service:local --name ai-infra-starter-kit -kind load docker-image ai-infra-starter-kit/ai-sre-assistant:local --name ai-infra-starter-kit +kind load docker-image aiops-lab/demo-service:local --name aiops-lab +kind load docker-image aiops-lab/ai-sre-assistant:local --name aiops-lab ``` ## 4. Apply Manifests @@ -112,14 +112,14 @@ kubectl apply -f infra/k8s/ai-sre-assistant.yaml ## 5. Wait For Apps ```bash -kubectl wait --for=condition=available deployment/demo-service -n ai-infra-starter-kit --timeout=120s -kubectl wait --for=condition=available deployment/ai-sre-assistant -n ai-infra-starter-kit --timeout=120s +kubectl wait --for=condition=available deployment/demo-service -n aiops-lab --timeout=120s +kubectl wait --for=condition=available deployment/ai-sre-assistant -n aiops-lab --timeout=120s ``` Inspect what is running: ```bash -kubectl get pods,svc,pvc -n ai-infra-starter-kit +kubectl get pods,svc,pvc -n aiops-lab ``` For a more complete operations and troubleshooting checklist, see `../../docs/10-kubernetes-operations-runbook.md`. @@ -135,13 +135,13 @@ For local-vs-production Kubernetes next steps, see `../../docs/14-kubernetes-pro Open one terminal for `demo-service`: ```bash -kubectl port-forward svc/demo-service 8000:8000 -n ai-infra-starter-kit +kubectl port-forward svc/demo-service 8000:8000 -n aiops-lab ``` Open another terminal for `ai-sre-assistant`: ```bash -kubectl port-forward svc/ai-sre-assistant 8001:8001 -n ai-infra-starter-kit +kubectl port-forward svc/ai-sre-assistant 8001:8001 -n aiops-lab ``` Now the services are available locally: @@ -188,14 +188,14 @@ curl -s -X POST http://localhost:8001/summarize-incident \ View logs: ```bash -kubectl logs deployment/demo-service -n ai-infra-starter-kit -kubectl logs deployment/ai-sre-assistant -n ai-infra-starter-kit +kubectl logs deployment/demo-service -n aiops-lab +kubectl logs deployment/ai-sre-assistant -n aiops-lab ``` Describe a pod if something is not ready: ```bash -kubectl describe pod -l app.kubernetes.io/name=demo-service -n ai-infra-starter-kit +kubectl describe pod -l app.kubernetes.io/name=demo-service -n aiops-lab ``` ## 11. Cleanup @@ -203,19 +203,19 @@ kubectl describe pod -l app.kubernetes.io/name=demo-service -n ai-infra-starter- Delete the namespace: ```bash -kubectl delete namespace ai-infra-starter-kit +kubectl delete namespace aiops-lab ``` Delete the local persistent volume if it remains: ```bash -kubectl delete pv ai-infra-shared-logs +kubectl delete pv aiops-shared-logs ``` Delete the kind cluster: ```bash -kind delete cluster --name ai-infra-starter-kit +kind delete cluster --name aiops-lab ``` ## Kubernetes Objects In Plain English diff --git a/infra/k8s/ai-sre-assistant.yaml b/infra/k8s/ai-sre-assistant.yaml index 630e1d6..d76e7e0 100644 --- a/infra/k8s/ai-sre-assistant.yaml +++ b/infra/k8s/ai-sre-assistant.yaml @@ -2,10 +2,10 @@ apiVersion: apps/v1 kind: Deployment metadata: name: ai-sre-assistant - namespace: ai-infra-starter-kit + namespace: aiops-lab labels: app.kubernetes.io/name: ai-sre-assistant - app.kubernetes.io/part-of: ai-infra-starter-kit + app.kubernetes.io/part-of: aiops-lab app.kubernetes.io/component: sre-assistant spec: replicas: 1 @@ -16,12 +16,12 @@ spec: metadata: labels: app.kubernetes.io/name: ai-sre-assistant - app.kubernetes.io/part-of: ai-infra-starter-kit + app.kubernetes.io/part-of: aiops-lab app.kubernetes.io/component: sre-assistant spec: containers: - name: ai-sre-assistant - image: ai-infra-starter-kit/ai-sre-assistant:local + image: aiops-lab/ai-sre-assistant:local imagePullPolicy: IfNotPresent ports: - name: http @@ -30,37 +30,37 @@ spec: - name: LLM_PROVIDER valueFrom: configMapKeyRef: - name: ai-infra-starter-kit-config + name: aiops-lab-config key: LLM_PROVIDER - name: OPENAI_BASE_URL valueFrom: configMapKeyRef: - name: ai-infra-starter-kit-config + name: aiops-lab-config key: OPENAI_BASE_URL - name: MODEL_NAME valueFrom: configMapKeyRef: - name: ai-infra-starter-kit-config + name: aiops-lab-config key: MODEL_NAME - name: LLM_MAX_LOG_ENTRIES valueFrom: configMapKeyRef: - name: ai-infra-starter-kit-config + name: aiops-lab-config key: LLM_MAX_LOG_ENTRIES - name: LLM_MAX_PROMPT_CHARS valueFrom: configMapKeyRef: - name: ai-infra-starter-kit-config + name: aiops-lab-config key: LLM_MAX_PROMPT_CHARS - name: DEMO_SERVICE_LOG_PATH valueFrom: configMapKeyRef: - name: ai-infra-starter-kit-config + name: aiops-lab-config key: DEMO_SERVICE_LOG_PATH - name: DEMO_SERVICE_METRICS_URL valueFrom: configMapKeyRef: - name: ai-infra-starter-kit-config + name: aiops-lab-config key: DEMO_SERVICE_METRICS_URL - name: OPENAI_API_KEY valueFrom: @@ -106,10 +106,10 @@ apiVersion: v1 kind: Service metadata: name: ai-sre-assistant - namespace: ai-infra-starter-kit + namespace: aiops-lab labels: app.kubernetes.io/name: ai-sre-assistant - app.kubernetes.io/part-of: ai-infra-starter-kit + app.kubernetes.io/part-of: aiops-lab app.kubernetes.io/component: sre-assistant spec: type: ClusterIP diff --git a/infra/k8s/configmap.yaml b/infra/k8s/configmap.yaml index e6d3b91..8b74da0 100644 --- a/infra/k8s/configmap.yaml +++ b/infra/k8s/configmap.yaml @@ -1,11 +1,11 @@ apiVersion: v1 kind: ConfigMap metadata: - name: ai-infra-starter-kit-config - namespace: ai-infra-starter-kit + name: aiops-lab-config + namespace: aiops-lab labels: - app.kubernetes.io/name: ai-infra-starter-kit-config - app.kubernetes.io/part-of: ai-infra-starter-kit + app.kubernetes.io/name: aiops-lab-config + app.kubernetes.io/part-of: aiops-lab data: # Safe-to-commit application configuration. Do not put API keys here. DEMO_SERVICE_LOG_PATH: /shared/logs/demo-service.log diff --git a/infra/k8s/demo-service.yaml b/infra/k8s/demo-service.yaml index dd54fce..a3966a4 100644 --- a/infra/k8s/demo-service.yaml +++ b/infra/k8s/demo-service.yaml @@ -2,10 +2,10 @@ apiVersion: apps/v1 kind: Deployment metadata: name: demo-service - namespace: ai-infra-starter-kit + namespace: aiops-lab labels: app.kubernetes.io/name: demo-service - app.kubernetes.io/part-of: ai-infra-starter-kit + app.kubernetes.io/part-of: aiops-lab app.kubernetes.io/component: demo-api spec: replicas: 1 @@ -16,12 +16,12 @@ spec: metadata: labels: app.kubernetes.io/name: demo-service - app.kubernetes.io/part-of: ai-infra-starter-kit + app.kubernetes.io/part-of: aiops-lab app.kubernetes.io/component: demo-api spec: containers: - name: demo-service - image: ai-infra-starter-kit/demo-service:local + image: aiops-lab/demo-service:local imagePullPolicy: IfNotPresent ports: - name: http @@ -30,7 +30,7 @@ spec: - name: DEMO_SERVICE_LOG_PATH valueFrom: configMapKeyRef: - name: ai-infra-starter-kit-config + name: aiops-lab-config key: DEMO_SERVICE_LOG_PATH volumeMounts: - name: demo-logs @@ -70,10 +70,10 @@ apiVersion: v1 kind: Service metadata: name: demo-service - namespace: ai-infra-starter-kit + namespace: aiops-lab labels: app.kubernetes.io/name: demo-service - app.kubernetes.io/part-of: ai-infra-starter-kit + app.kubernetes.io/part-of: aiops-lab app.kubernetes.io/component: demo-api spec: type: ClusterIP diff --git a/infra/k8s/namespace.yaml b/infra/k8s/namespace.yaml index 0c908d1..2a07dac 100644 --- a/infra/k8s/namespace.yaml +++ b/infra/k8s/namespace.yaml @@ -1,8 +1,8 @@ apiVersion: v1 kind: Namespace metadata: - name: ai-infra-starter-kit + name: aiops-lab labels: - app.kubernetes.io/name: ai-infra-starter-kit - app.kubernetes.io/part-of: ai-infra-starter-kit + app.kubernetes.io/name: aiops-lab + app.kubernetes.io/part-of: aiops-lab diff --git a/infra/k8s/secret.example.yaml b/infra/k8s/secret.example.yaml index 73d9301..21f9904 100644 --- a/infra/k8s/secret.example.yaml +++ b/infra/k8s/secret.example.yaml @@ -2,10 +2,10 @@ apiVersion: v1 kind: Secret metadata: name: ai-sre-assistant-secrets - namespace: ai-infra-starter-kit + namespace: aiops-lab labels: app.kubernetes.io/name: ai-sre-assistant-secrets - app.kubernetes.io/part-of: ai-infra-starter-kit + app.kubernetes.io/part-of: aiops-lab type: Opaque stringData: # Example only. Leave empty for the default rule-based assistant. diff --git a/infra/k8s/storage.yaml b/infra/k8s/storage.yaml index 65a9e66..0413b5c 100644 --- a/infra/k8s/storage.yaml +++ b/infra/k8s/storage.yaml @@ -1,34 +1,34 @@ apiVersion: v1 kind: PersistentVolume metadata: - name: ai-infra-shared-logs + name: aiops-shared-logs labels: - app.kubernetes.io/name: ai-infra-shared-logs - app.kubernetes.io/part-of: ai-infra-starter-kit + app.kubernetes.io/name: aiops-shared-logs + app.kubernetes.io/part-of: aiops-lab spec: capacity: storage: 1Gi accessModes: - ReadWriteOnce persistentVolumeReclaimPolicy: Delete - storageClassName: ai-infra-local + storageClassName: aiops-local hostPath: - path: /var/local/ai-infra-starter-kit/logs + path: /var/local/aiops-lab/logs type: DirectoryOrCreate --- apiVersion: v1 kind: PersistentVolumeClaim metadata: name: demo-service-logs - namespace: ai-infra-starter-kit + namespace: aiops-lab labels: app.kubernetes.io/name: demo-service-logs - app.kubernetes.io/part-of: ai-infra-starter-kit + app.kubernetes.io/part-of: aiops-lab spec: accessModes: - ReadWriteOnce - storageClassName: ai-infra-local - volumeName: ai-infra-shared-logs + storageClassName: aiops-local + volumeName: aiops-shared-logs resources: requests: storage: 1Gi diff --git a/scripts/generate-demo-traffic.py b/scripts/generate-demo-traffic.py index 3451013..1991181 100644 --- a/scripts/generate-demo-traffic.py +++ b/scripts/generate-demo-traffic.py @@ -7,7 +7,7 @@ def call(url: str) -> tuple[int, str]: - request = urllib.request.Request(url, headers={"User-Agent": "ai-infra-starter-kit-traffic/0.1"}) + request = urllib.request.Request(url, headers={"User-Agent": "aiops-lab-traffic/0.1"}) try: with urllib.request.urlopen(request, timeout=10) as response: body = response.read().decode("utf-8")