diff --git a/analytics/app/cell-birthdays/page.tsx b/analytics/app/cell-birthdays/page.tsx new file mode 100644 index 00000000..2b74d918 --- /dev/null +++ b/analytics/app/cell-birthdays/page.tsx @@ -0,0 +1,78 @@ +'use client'; + +import CellAgeMap from '@/components/charts/CellAgeMap'; +import { exportRowsToCsv } from '@/lib/export'; +import ExportButton from '@/components/ui/ExportButton'; + +const sampleCells = [ + { geohash: 'u09tvq0', lat: 40.7128, lng: -74.0060, gistCount: 2841, ageDays: 3, lastActivity: '2026-07-28' }, + { geohash: 'u09tvq1', lat: 40.7130, lng: -74.0058, gistCount: 1562, ageDays: 12, lastActivity: '2026-07-20' }, + { geohash: 'u09tvq2', lat: 40.7132, lng: -74.0056, gistCount: 892, ageDays: 45, lastActivity: '2026-06-15' }, + { geohash: 'u09tvq3', lat: 40.7126, lng: -74.0062, gistCount: 3421, ageDays: 1, lastActivity: '2026-07-29' }, + { geohash: 'u09tvq4', lat: 40.7124, lng: -74.0064, gistCount: 723, ageDays: 89, lastActivity: '2026-05-01' }, + { geohash: 'u09tvq5', lat: 40.7134, lng: -74.0054, gistCount: 456, ageDays: 180, lastActivity: '2026-01-30' }, + { geohash: 'u09tvq6', lat: 40.7118, lng: -74.0070, gistCount: 98, ageDays: 365, lastActivity: '2025-07-30' }, + { geohash: 'u09tvq7', lat: 40.7140, lng: -74.0050, gistCount: 2100, ageDays: 7, lastActivity: '2026-07-22' }, + { geohash: 'u09tvq8', lat: 40.7120, lng: -74.0068, gistCount: 567, ageDays: 30, lastActivity: '2026-06-29' }, + { geohash: 'u09tvq9', lat: 40.7138, lng: -74.0052, gistCount: 1890, ageDays: 60, lastActivity: '2026-05-30' }, + { geohash: 'u09tvr0', lat: 40.7150, lng: -74.0040, gistCount: 3045, ageDays: 2, lastActivity: '2026-07-27' }, + { geohash: 'u09tvr1', lat: 40.7148, lng: -74.0042, gistCount: 1234, ageDays: 14, lastActivity: '2026-07-15' }, + { geohash: 'u09tvr2', lat: 40.7146, lng: -74.0044, gistCount: 678, ageDays: 90, lastActivity: '2026-04-30' }, + { geohash: 'u09tvr3', lat: 40.7152, lng: -74.0038, gistCount: 2345, ageDays: 5, lastActivity: '2026-07-24' }, + { geohash: 'u09tvr4', lat: 40.7144, lng: -74.0046, gistCount: 432, ageDays: 200, lastActivity: '2026-01-10' }, + { geohash: 'u09tvr5', lat: 40.7154, lng: -74.0036, gistCount: 1567, ageDays: 21, lastActivity: '2026-07-08' }, + { geohash: 'u09tvr6', lat: 40.7136, lng: -74.0058, gistCount: 890, ageDays: 150, lastActivity: '2026-03-01' }, + { geohash: 'u09tvr7', lat: 40.7142, lng: -74.0048, gistCount: 210, ageDays: 400, lastActivity: '2025-06-24' }, + { geohash: 'u09tvr8', lat: 40.7156, lng: -74.0034, gistCount: 2789, ageDays: 4, lastActivity: '2026-07-25' }, + { geohash: 'u09tvr9', lat: 40.7135, lng: -74.0055, gistCount: 3456, ageDays: 0, lastActivity: '2026-07-29' }, + { geohash: 'u09tvs0', lat: 34.0522, lng: -118.2437, gistCount: 1890, ageDays: 10, lastActivity: '2026-07-19' }, + { geohash: 'u09tvs1', lat: 34.0524, lng: -118.2435, gistCount: 2340, ageDays: 6, lastActivity: '2026-07-23' }, + { geohash: 'u09tvs2', lat: 34.0520, lng: -118.2439, gistCount: 567, ageDays: 75, lastActivity: '2026-05-15' }, + { geohash: 'u09tvs3', lat: 34.0526, lng: -118.2433, gistCount: 1234, ageDays: 20, lastActivity: '2026-07-09' }, + { geohash: 'u09tvs4', lat: 34.0518, lng: -118.2441, gistCount: 345, ageDays: 250, lastActivity: '2025-11-22' }, + { geohash: 'u09tvs5', lat: 51.5074, lng: -0.1278, gistCount: 4567, ageDays: 1, lastActivity: '2026-07-28' }, + { geohash: 'u09tvs6', lat: 51.5076, lng: -0.1276, gistCount: 2890, ageDays: 8, lastActivity: '2026-07-21' }, + { geohash: 'u09tvs7', lat: 51.5072, lng: -0.1280, gistCount: 1678, ageDays: 35, lastActivity: '2026-06-24' }, + { geohash: 'u09tvs8', lat: 51.5078, lng: -0.1274, gistCount: 901, ageDays: 120, lastActivity: '2026-03-31' }, + { geohash: 'u09tvs9', lat: 51.5070, lng: -0.1282, gistCount: 234, ageDays: 300, lastActivity: '2025-10-03' }, +]; + +export default function CellBirthdaysPage() { + return ( +
+
+
Cell Birthdays
+
+
+

Geohash Cell Birthday Tracker

+

+ Track the age and activity of geohash cells across the GistPin platform. Identify aging cells with declining activity. +

+
+ + exportRowsToCsv({ + filenamePrefix: 'cell-birthdays', + rows: sampleCells.map((c) => ({ geohash: c.geohash, lat: c.lat, lng: c.lng, age_days: c.ageDays, gist_count: c.gistCount, last_activity: c.lastActivity })), + onProgress, + }) + } + /> +
+
+ +
+ +
+
+ ); +} diff --git a/analytics/components/charts/CellAgeMap.tsx b/analytics/components/charts/CellAgeMap.tsx new file mode 100644 index 00000000..7a671de7 --- /dev/null +++ b/analytics/components/charts/CellAgeMap.tsx @@ -0,0 +1,215 @@ +'use client'; + +import { useMemo, useState } from 'react'; + +interface GeohashCell { + geohash: string; + lat: number; + lng: number; + gistCount: number; + ageDays: number; + lastActivity: string; +} + +interface CellAgeMapProps { + cells: GeohashCell[]; +} + +function geohashToBounds(geohash: string): { minLat: number; maxLat: number; minLng: number; maxLng: number } { + const BASE32 = '0123456789bcdefghjkmnpqrstuvwxyz'; + let minLat = -90; let maxLat = 90; + let minLng = -180; let maxLng = 180; + let isEven = true; + + for (const char of geohash) { + const idx = BASE32.indexOf(char); + for (let i = 4; i >= 0; i--) { + const bit = (idx >> i) & 1; + if (isEven) { + const mid = (minLng + maxLng) / 2; + if (bit === 1) minLng = mid; + else maxLng = mid; + } else { + const mid = (minLat + maxLat) / 2; + if (bit === 1) minLat = mid; + else maxLat = mid; + } + isEven = !isEven; + } + } + return { minLat, maxLat, minLng, maxLng }; +} + +function ageColor(ageDays: number): string { + if (ageDays <= 7) return '#22c55e'; + if (ageDays <= 30) return '#84cc16'; + if (ageDays <= 90) return '#eab308'; + if (ageDays <= 180) return '#f97316'; + if (ageDays <= 365) return '#ef4444'; + return '#7c3aed'; +} + +function ageLabel(ageDays: number): string { + if (ageDays <= 7) return '< 1 week'; + if (ageDays <= 30) return '< 1 month'; + if (ageDays <= 90) return '< 3 months'; + if (ageDays <= 180) return '< 6 months'; + if (ageDays <= 365) return '< 1 year'; + return '> 1 year'; +} + +export default function CellAgeMap({ cells }: CellAgeMapProps) { + const [minAge, setMinAge] = useState(0); + const [maxAge, setMaxAge] = useState(Infinity); + const [sortBy, setSortBy] = useState<'age' | 'count'>('age'); + + const filteredCells = useMemo( + () => cells + .filter((c) => c.ageDays >= minAge && c.ageDays <= maxAge) + .sort((a, b) => sortBy === 'age' ? b.ageDays - a.ageDays : b.gistCount - a.gistCount), + [cells, minAge, maxAge, sortBy] + ); + + const stats = useMemo(() => { + if (cells.length === 0) return null; + const ages = cells.map((c) => c.ageDays); + return { + totalCells: cells.length, + avgAge: Math.round(ages.reduce((s, v) => s + v, 0) / ages.length), + maxAge: Math.max(...ages), + minAge: Math.min(...ages), + totalGists: cells.reduce((s, c) => s + c.gistCount, 0), + }; + }, [cells]); + + return ( +
+ {stats && ( +
+ {[ + { label: 'Active Cells', value: stats.totalCells }, + { label: 'Avg Age', value: `${stats.avgAge} days` }, + { label: 'Oldest Cell', value: `${stats.maxAge} days` }, + { label: 'Total Gists', value: stats.totalGists.toLocaleString() }, + ].map((s) => ( +
+
{s.label}
+
{s.value}
+
+ ))} +
+ )} + +
+
+ + setMinAge(Number(e.target.value))} style={{ padding: '4px 8px', borderRadius: 6, border: '1px solid #e2e8f0', width: 70, fontSize: 13 }} /> +
+
+ + setMaxAge(e.target.value ? Number(e.target.value) : Infinity)} style={{ padding: '4px 8px', borderRadius: 6, border: '1px solid #e2e8f0', width: 70, fontSize: 13 }} /> +
+
+ {(['age', 'count'] as const).map((s) => ( + + ))} +
+
+ +
+ {[ + { label: '< 1 week', color: '#22c55e', min: 0, max: 7 }, + { label: '< 1 month', color: '#84cc16', min: 8, max: 30 }, + { label: '< 3 months', color: '#eab308', min: 31, max: 90 }, + { label: '< 6 months', color: '#f97316', min: 91, max: 180 }, + { label: '< 1 year', color: '#ef4444', min: 181, max: 365 }, + { label: '> 1 year', color: '#7c3aed', min: 366, max: Infinity }, + ].map((bucket) => { + const count = filteredCells.filter((c) => c.ageDays >= bucket.min && c.ageDays <= bucket.max).length; + return ( +
+ + {bucket.label} + {count} +
+ ); + })} +
+ +
+ {filteredCells.slice(0, 100).map((cell) => { + const bounds = geohashToBounds(cell.geohash); + const centerLat = ((bounds.minLat + bounds.maxLat) / 2).toFixed(4); + const centerLng = ((bounds.minLng + bounds.maxLng) / 2).toFixed(4); + return ( +
+
+ {cell.geohash} + + ({centerLat}, {centerLng}) + +
+
+ + {cell.ageDays}d — {ageLabel(cell.ageDays)} +
+
{cell.gistCount.toLocaleString()} gists
+
{cell.lastActivity}
+
+ ); + })} +
+ + {filteredCells.length > 100 && ( +
+ Showing 100 of {filteredCells.length} cells. Refine filters to narrow results. +
+ )} + + {filteredCells.length === 0 && ( +
+ No geohash cells match the current filters. +
+ )} +
+ ); +} diff --git a/infrastructure/docs/garbage-collection.md b/infrastructure/docs/garbage-collection.md new file mode 100644 index 00000000..399c4478 --- /dev/null +++ b/infrastructure/docs/garbage-collection.md @@ -0,0 +1,70 @@ +# Kubernetes Garbage Collection Tuning + +Kubelet garbage collection configuration for GistPin Kubernetes clusters. + +## Overview + +Proper garbage collection tuning prevents node pressure, evictions, and performance degradation. The configuration in `infrastructure/k8s/kubelet-gc-config.yaml` defines thresholds for image GC, container log rotation, and eviction policies. + +## Image Garbage Collection + +Images are garbage collected when disk usage exceeds the high threshold: + +| Parameter | Value | Description | +|-----------|-------|-------------| +| `imageGCHighThresholdPercent` | 85% | Trigger image GC above this usage | +| `imageGCLowThresholdPercent` | 80% | Stop image GC at this usage | +| `imageMinimumGCAge` | 2m | Minimum age before image removal | + +## Container Log Rotation + +| Parameter | Value | Description | +|-----------|-------|-------------| +| `containerLogMaxSize` | 50Mi | Max log file size before rotation | +| `containerLogMaxFiles` | 5 | Max log files to retain per container | + +## Eviction Thresholds + +### Hard Eviction (immediate pod termination) + +| Signal | Threshold | +|--------|-----------| +| `memory.available` | < 500Mi | +| `nodefs.available` | < 10% | +| `nodefs.inodesFree` | < 5% | +| `imagefs.available` | < 15% | + +### Soft Eviction (graceful termination) + +| Signal | Threshold | Grace Period | +|--------|-----------|--------------| +| `memory.available` | < 800Mi | 1m30s | +| `nodefs.available` | < 15% | 2m | +| `nodefs.inodesFree` | < 10% | 2m | +| `imagefs.available` | < 20% | 2m | + +## Reserved Resources + +Resources reserved for system and kubelet daemons: + +| Cgroup | CPU | Memory | Ephemeral Storage | +|--------|-----|--------|-------------------| +| kubelet | 200m | 512Mi | 2Gi | +| system | 100m | 256Mi | 1Gi | + +## Alerting + +Alerting rules in `infrastructure/monitoring/gc-alerts.yml`: + +| Alert | Severity | Description | +|-------|----------|-------------| +| GCHighImageUsage | warning | Image FS > 85% | +| GCEvictionThreshold | critical | Eviction signal breached | +| GCMemoryPressure | warning | Memory pressure | +| GCDiskPressure | warning | Disk pressure | + +## Related Resources + +- [Kubelet GC Config](../k8s/kubelet-gc-config.yaml) +- [GC Alerting Rules](../monitoring/gc-alerts.yml) +- [Node Alerts](../monitoring/node-alerts.yml) diff --git a/infrastructure/docs/tagging-taxonomy.md b/infrastructure/docs/tagging-taxonomy.md new file mode 100644 index 00000000..69f10ac0 --- /dev/null +++ b/infrastructure/docs/tagging-taxonomy.md @@ -0,0 +1,66 @@ +# Tagging Taxonomy + +Standardized resource tagging strategy for GistPin infrastructure. + +## Taxonomy Categories + +| Category | Prefix | Purpose | +|----------|--------|---------| +| Automation | `gistpin:managed-by`, `gistpin:provisioner` | Provisioning metadata | +| Cost Allocation | `gistpin:cost-center`, `gistpin:project`, `gistpin:service` | Billing and cost tracking | +| Ownership | `gistpin:owner`, `gistpin:team`, `gistpin:contact-email` | Resource responsibility | +| Operational | `gistpin:tier`, `gistpin:criticality`, `gistpin:backup` | Runbook and operations | +| Security | `gistpin:data-classification`, `gistpin:compliance` | Security posture | +| Lifecycle | `gistpin:created-by`, `gistpin:decommission-date` | Resource lifecycle | + +## Tier Definitions + +| Tier | Name | Backup | Retention | Classification | +|------|------|--------|-----------|----------------| +| 1 | Critical | Daily | 365 days | Restricted | +| 2 | Important | Daily | 90 days | Sensitive | +| 3 | Standard | Weekly | 30 days | Internal | +| 4 | Dev/Test | None | 7 days | Public | + +## Usage + +Apply tier-based tags in Terraform: + +```hcl +resource "aws_s3_bucket" "data" { + bucket = "gistpin-data" + tags = local.tier1_critical_tags +} +``` + +For service-specific tags, merge with service map: + +```hcl +resource "aws_s3_bucket" "analytics" { + bucket = "gistpin-analytics" + tags = merge(local.tier3_standard_tags, local.service_tags["analytics"]) +} +``` + +## Enforcement + +Tagging compliance is enforced via: +1. **Sentinel policy** – `infrastructure/terraform/tagging-taxonomy.tf` deploys the policy +2. **AWS Config rules** – `required-tags` and `gistpin-tagging-taxonomy` rules +3. **CI/CD gating** – Terraform Cloud policy checks on every plan + +## Verification + +```bash +# List resources missing required tags +aws resourcegroupstaggingapi get-resources --tag-filters Key=gistpin:environment + +# Check Config compliance +aws configservice describe-compliance-by-config-rule --config-rule-names gistpin-tagging-taxonomy +``` + +## Related Resources + +- [Tag Taxonomy Terraform](../terraform/tagging-taxonomy.tf) +- [Tag Locals](../terraform/tag-locals.tf) +- [Tag Policy (legacy)](../terraform/tag-policy.tf) diff --git a/infrastructure/k8s/boundary/controller.yaml b/infrastructure/k8s/boundary/controller.yaml new file mode 100644 index 00000000..2d006ca9 --- /dev/null +++ b/infrastructure/k8s/boundary/controller.yaml @@ -0,0 +1,191 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: boundary + labels: + app: boundary + component: controller + gistpin:tier: "2" +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: boundary-controller + namespace: boundary + labels: + app: boundary + component: controller +--- +apiVersion: v1 +kind: Secret +metadata: + name: boundary-controller-config + namespace: boundary + labels: + app: boundary + component: controller +stringData: + config.hcl: | + controller { + name = "gistpin-boundary-controller" + description = "GistPin Boundary Controller" + } + + api { + listener { + purpose = "api" + address = "0.0.0.0:9200" + tls_disable = true + } + } + + cluster { + listener { + purpose = "cluster" + address = "0.0.0.0:9201" + tls_disable = true + } + } + + worker { + listener { + purpose = "proxy" + address = "0.0.0.0:9202" + tls_disable = true + } + } + + events { + audit_enabled = true + observations_enabled = true + sysevents_enabled = true + sink { + type = "stderr" + format = "json" + } + } + + kms "aead" { + purpose = "root" + aead_type = "aes-gcm" + key = "7xtHkPqYmF2zN3L4v5R6s8tA0bC1dE9f" + } + + kms "aead" { + purpose = "worker-auth" + aead_type = "aes-gcm" + key = "8yuIkLrZnW3oP4qR5sT6uV7wX0yA2bC3" + } + + kms "aead" { + purpose = "recovery" + aead_type = "aes-gcm" + key = "9zWjMnBvCx5lK2oP3qR4sT6uV8wY0aD1" + } +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: boundary-controller + namespace: boundary + labels: + app: boundary + component: controller +spec: + serviceName: boundary-controller + replicas: 3 + selector: + matchLabels: + app: boundary + component: controller + template: + metadata: + labels: + app: boundary + component: controller + spec: + serviceAccountName: boundary-controller + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app: boundary + component: controller + topologyKey: kubernetes.io/hostname + containers: + - name: controller + image: hashicorp/boundary:0.15.0 + args: + - server + - -config=/etc/boundary/config.hcl + ports: + - containerPort: 9200 + name: api + - containerPort: 9201 + name: cluster + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: config + mountPath: /etc/boundary + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 500m + memory: 1Gi + livenessProbe: + httpGet: + path: /health + port: 9200 + initialDelaySeconds: 30 + periodSeconds: 15 + readinessProbe: + httpGet: + path: /health + port: 9200 + initialDelaySeconds: 5 + periodSeconds: 5 + volumes: + - name: config + secret: + secretName: boundary-controller-config +--- +apiVersion: v1 +kind: Service +metadata: + name: boundary-controller + namespace: boundary + labels: + app: boundary + component: controller +spec: + ports: + - port: 9200 + name: api + targetPort: 9200 + - port: 9201 + name: cluster + targetPort: 9201 + selector: + app: boundary + component: controller +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: boundary-controller-pdb + namespace: boundary +spec: + minAvailable: 2 + selector: + matchLabels: + app: boundary + component: controller diff --git a/infrastructure/k8s/boundary/worker.yaml b/infrastructure/k8s/boundary/worker.yaml new file mode 100644 index 00000000..992c5d3a --- /dev/null +++ b/infrastructure/k8s/boundary/worker.yaml @@ -0,0 +1,122 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: boundary-worker + namespace: boundary + labels: + app: boundary + component: worker +--- +apiVersion: v1 +kind: Secret +metadata: + name: boundary-worker-config + namespace: boundary + labels: + app: boundary + component: worker +stringData: + config.hcl: | + worker { + name = "gistpin-boundary-worker-${HOSTNAME}" + description = "GistPin Boundary Worker" + initial_upstreams = ["boundary-controller.boundary:9201"] + } + + listener { + purpose = "proxy" + address = "0.0.0.0:9202" + tls_disable = true + } + + controller { + name = "gistpin-boundary-controller" + } + + kms "aead" { + purpose = "worker-auth" + aead_type = "aes-gcm" + key = "8yuIkLrZnW3oP4qR5sT6uV7wX0yA2bC3" + } + + events { + audit_enabled = true + observations_enabled = true + sink { + type = "stderr" + format = "json" + } + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: boundary-worker + namespace: boundary + labels: + app: boundary + component: worker +spec: + selector: + matchLabels: + app: boundary + component: worker + template: + metadata: + labels: + app: boundary + component: worker + spec: + serviceAccountName: boundary-worker + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: worker + image: hashicorp/boundary:0.15.0 + args: + - server + - -config=/etc/boundary/config.hcl + ports: + - containerPort: 9202 + name: proxy + env: + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: config + mountPath: /etc/boundary + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 200m + memory: 512Mi + livenessProbe: + tcpSocket: + port: 9202 + initialDelaySeconds: 15 + periodSeconds: 10 + volumes: + - name: config + secret: + secretName: boundary-worker-config +--- +apiVersion: v1 +kind: Service +metadata: + name: boundary-worker + namespace: boundary + labels: + app: boundary + component: worker +spec: + ports: + - port: 9202 + name: proxy + targetPort: 9202 + selector: + app: boundary + component: worker diff --git a/infrastructure/k8s/kubelet-gc-config.yaml b/infrastructure/k8s/kubelet-gc-config.yaml new file mode 100644 index 00000000..e9c7e05d --- /dev/null +++ b/infrastructure/k8s/kubelet-gc-config.yaml @@ -0,0 +1,52 @@ +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +metadata: + name: kubelet-gc-config + namespace: gistpin-prod +imageGCHighThresholdPercent: 85 +imageGCLowThresholdPercent: 80 +evictionHard: + memory.available: "500Mi" + nodefs.available: "10%" + nodefs.inodesFree: "5%" + imagefs.available: "15%" +evictionSoft: + memory.available: "800Mi" + nodefs.available: "15%" + nodefs.inodesFree: "10%" + imagefs.available: "20%" +evictionSoftGracePeriod: + memory.available: "1m30s" + nodefs.available: "2m" + nodefs.inodesFree: "2m" + imagefs.available: "2m" +evictionPressureTransitionPeriod: "2m" +evictionMaxPodGracePeriod: 120 +evictionMinimumReclaim: + memory.available: "256Mi" + nodefs.available: "5%" + imagefs.available: "5%" +imageMinimumGCAge: 2m +maxPerPodContainerCount: 3 +maxContainerCount: -1 +containerLogMaxSize: "50Mi" +containerLogMaxFiles: 5 +eventRecordQPS: 10 +eventBurst: 20 +kubeReserved: + cpu: "200m" + memory: "512Mi" + ephemeral-storage: "2Gi" +systemReserved: + cpu: "100m" + memory: "256Mi" + ephemeral-storage: "1Gi" +kubeReservedCgroup: "/kubelet" +systemReservedCgroup: "/system" +enforceNodeAllocatable: + - pods + - kube-reserved + - system-reserved +cpuManagerPolicy: static +topologyManagerPolicy: best-effort +memoryManagerPolicy: Static diff --git a/infrastructure/monitoring/gc-alerts.yml b/infrastructure/monitoring/gc-alerts.yml new file mode 100644 index 00000000..9dcef1d6 --- /dev/null +++ b/infrastructure/monitoring/gc-alerts.yml @@ -0,0 +1,66 @@ +groups: + - name: garbage-collection + interval: 30s + rules: + - alert: GCHighImageUsage + expr: kubelet_image_fs_usage_bytes / kubelet_image_fs_capacity_bytes > 0.85 + for: 5m + labels: + severity: warning + annotations: + summary: "Image filesystem usage above GC threshold" + description: "Image FS usage at {{ $value | humanizePercentage }} on {{ $labels.node }}" + + - alert: GCEvictionThreshold + expr: kubelet_eviction_threshold_reached > 0 + for: 0s + labels: + severity: critical + annotations: + summary: "Kubelet eviction threshold reached on {{ $labels.node }}" + description: "{{ $labels.eviction_signal }} exceeded on node {{ $labels.node }}" + + - alert: GCMemoryPressure + expr: kubelet_node_memory_pressure > 0 + for: 2m + labels: + severity: warning + annotations: + summary: "Memory pressure on node {{ $labels.node }}" + description: "Node {{ $labels.node }} is under memory pressure" + + - alert: GCDiskPressure + expr: kubelet_node_disk_pressure > 0 + for: 2m + labels: + severity: warning + annotations: + summary: "Disk pressure on node {{ $labels.node }}" + description: "Node {{ $labels.node }} is under disk pressure" + + - alert: GCPidPressure + expr: kubelet_node_pid_pressure > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "PID pressure on node {{ $labels.node }}" + description: "Node {{ $labels.node }} is under PID pressure" + + - alert: GCContainerLogSize + expr: avg by (node) (kubelet_container_log_filesystem_usage_bytes) > 4e7 + for: 10m + labels: + severity: info + annotations: + summary: "Container logs growing large on {{ $labels.node }}" + description: "Average container log size is {{ $value | humanize1024 }}" + + - alert: GCImagePullFailures + expr: rate(kubelet_image_pull_failure_total[10m]) > 0.1 + for: 5m + labels: + severity: warning + annotations: + summary: "Elevated image pull failures on {{ $labels.node }}" + description: "{{ $value }} failures/second over 10 minutes" diff --git a/infrastructure/terraform/boundary-targets.tf b/infrastructure/terraform/boundary-targets.tf new file mode 100644 index 00000000..549d9903 --- /dev/null +++ b/infrastructure/terraform/boundary-targets.tf @@ -0,0 +1,167 @@ +################################################################################ +# Boundary Targets – secure remote access via HashiCorp Boundary +################################################################################ + +variable "boundary_controller_address" { + description = "Boundary controller API address" + type = string + default = "http://boundary-controller.boundary:9200" +} + +variable "boundary_auth_method_id" { + description = "Boundary auth method ID for target management" + type = string + default = "" +} + +locals { + boundary_tags = { + Environment = var.environment + Project = var.project_name + Component = "boundary-targets" + } +} + +resource "boundary_scope" "project" { + name = "${var.project_name}-${var.environment}" + description = "GistPin ${var.environment} project scope" + scope_id = "global" + auto_create_admin_role = true + auto_create_default_role = true +} + +resource "boundary_auth_method" "password" { + name = "${var.project_name}-${var.environment}-auth" + type = "password" + scope_id = boundary_scope.project.id +} + +resource "boundary_account" "password_admin" { + name = "boundary-admin-${var.environment}" + type = "password" + login_name = "admin" + password = var.boundary_admin_password + auth_method_id = boundary_auth_method.password.id +} + +resource "boundary_user" "admin" { + name = "admin-${var.environment}" + scope_id = boundary_scope.project.id +} + +resource "boundary_role" "admin" { + name = "admin-role-${var.environment}" + description = "Admin role with full access" + scope_id = boundary_scope.project.id + grant_strings = [ + "id=*;type=*;actions=*" + ] + principal_ids = [boundary_user.admin.id] +} + +# ── SSH target for bastion hosts ──────────────────────────────────────── +resource "boundary_target" "ssh_bastion" { + name = "${var.project_name}-${var.environment}-ssh-bastion" + description = "SSH access to EC2 bastion hosts" + type = "ssh" + scope_id = boundary_scope.project.id + default_port = 22 + host_source_ids = [ + boundary_host_set.ec2_bastion.id + ] + brokered_credential_source_ids = [] +} + +resource "boundary_host_set" "ec2_bastion" { + name = "${var.project_name}-${var.environment}-ec2-bastion" + description = "EC2 bastion hosts" + type = "static" + scope_id = boundary_scope.project.id +} + +resource "boundary_host" "ec2_bastion_1" { + name = "${var.project_name}-${var.environment}-bastion-1" + type = "static" + scope_id = boundary_scope.project.id + host_set_id = boundary_host_set.ec2_bastion.id + address = var.bastion_host_1_address +} + +resource "boundary_host" "ec2_bastion_2" { + name = "${var.project_name}-${var.environment}-bastion-2" + type = "static" + scope_id = boundary_scope.project.id + host_set_id = boundary_host_set.ec2_bastion.id + address = var.bastion_host_2_address +} + +# ── PostgreSQL target ─────────────────────────────────────────────────── +resource "boundary_target" "postgres" { + name = "${var.project_name}-${var.environment}-postgres" + description = "PostgreSQL database access" + type = "tcp" + scope_id = boundary_scope.project.id + default_port = 5432 + host_source_ids = [ + boundary_host_set.postgres.id + ] +} + +resource "boundary_host_set" "postgres" { + name = "${var.project_name}-${var.environment}-postgres" + description = "PostgreSQL RDS instances" + type = "static" + scope_id = boundary_scope.project.id +} + +resource "boundary_host" "postgres_primary" { + name = "${var.project_name}-${var.environment}-postgres-primary" + type = "static" + scope_id = boundary_scope.project.id + host_set_id = boundary_host_set.postgres.id + address = var.postgres_host_address +} + +# ── Kubernetes API target ─────────────────────────────────────────────── +resource "boundary_target" "kubernetes_api" { + name = "${var.project_name}-${var.environment}-kubernetes-api" + description = "Kubernetes API server access" + type = "tcp" + scope_id = boundary_scope.project.id + default_port = 443 + host_source_ids = [ + boundary_host_set.kubernetes_api.id + ] +} + +resource "boundary_host_set" "kubernetes_api" { + name = "${var.project_name}-${var.environment}-kubernetes-api" + description = "Kubernetes API endpoints" + type = "static" + scope_id = boundary_scope.project.id +} + +resource "boundary_host" "kubernetes_api_endpoint" { + name = "${var.project_name}-${var.environment}-k8s-api" + type = "static" + scope_id = boundary_scope.project.id + host_set_id = boundary_host_set.kubernetes_api.id + address = var.kubernetes_api_endpoint +} + +# ── Outputs ───────────────────────────────────────────────────────────── +output "boundary_project_id" { + value = boundary_scope.project.id +} + +output "boundary_target_ssh_id" { + value = boundary_target.ssh_bastion.id +} + +output "boundary_target_postgres_id" { + value = boundary_target.postgres.id +} + +output "boundary_target_k8s_id" { + value = boundary_target.kubernetes_api.id +} diff --git a/infrastructure/terraform/tag-locals.tf b/infrastructure/terraform/tag-locals.tf new file mode 100644 index 00000000..1c518b4f --- /dev/null +++ b/infrastructure/terraform/tag-locals.tf @@ -0,0 +1,97 @@ +################################################################################ +# Tag Locals – consolidated tag maps for resource-level assignment +################################################################################ + +locals { + # ── Tier-based tag maps ────────────────────────────────────────────── + + tier1_critical_tags = { + "gistpin:environment" = var.environment + "gistpin:cost-center" = var.cost_center + "gistpin:project" = var.project_name + "gistpin:service" = "" + "gistpin:owner" = var.owner + "gistpin:team" = "platform-core" + "gistpin:tier" = "1" + "gistpin:criticality" = "critical" + "gistpin:backup" = "daily" + "gistpin:retention" = "365" + "gistpin:data-classification" = "restricted" + "gistpin:encryption-required" = "true" + "gistpin:audit-enabled" = "true" + "gistpin:managed-by" = "terraform" + "gistpin:provisioner" = "terraform-cloud" + } + + tier2_important_tags = { + "gistpin:environment" = var.environment + "gistpin:cost-center" = var.cost_center + "gistpin:project" = var.project_name + "gistpin:service" = "" + "gistpin:owner" = var.owner + "gistpin:team" = "" + "gistpin:tier" = "2" + "gistpin:criticality" = "high" + "gistpin:backup" = "daily" + "gistpin:retention" = "90" + "gistpin:data-classification" = "sensitive" + "gistpin:encryption-required" = "true" + "gistpin:audit-enabled" = "true" + "gistpin:managed-by" = "terraform" + "gistpin:provisioner" = "terraform-cloud" + } + + tier3_standard_tags = { + "gistpin:environment" = var.environment + "gistpin:cost-center" = var.cost_center + "gistpin:project" = var.project_name + "gistpin:service" = "" + "gistpin:owner" = var.owner + "gistpin:tier" = "3" + "gistpin:criticality" = "standard" + "gistpin:backup" = "weekly" + "gistpin:retention" = "30" + "gistpin:data-classification" = "internal" + "gistpin:managed-by" = "terraform" + "gistpin:provisioner" = "terraform-cloud" + } + + tier4_dev_tags = { + "gistpin:environment" = var.environment + "gistpin:cost-center" = var.cost_center + "gistpin:project" = var.project_name + "gistpin:owner" = var.owner + "gistpin:tier" = "4" + "gistpin:criticality" = "low" + "gistpin:backup" = "none" + "gistpin:retention" = "7" + "gistpin:data-classification" = "public" + "gistpin:managed-by" = "terraform" + "gistpin:provisioner" = "terraform-cloud" + } + + # ── Environment-specific tag overrides ─────────────────────────────── + + env_tags = { + production = merge(local.tier2_important_tags, { "gistpin:environment" = "production" }) + staging = merge(local.tier3_standard_tags, { "gistpin:environment" = "staging" }) + development = merge(local.tier4_dev_tags, { "gistpin:environment" = "development" }) + } + + # ── Service-specific tags ──────────────────────────────────────────── + + service_tags = { + api = { "gistpin:service" = "api", "gistpin:sub-service" = "backend" } + frontend = { "gistpin:service" = "frontend", "gistpin:sub-service" = "web" } + database = { "gistpin:service" = "database", "gistpin:sub-service" = "postgres" } + cache = { "gistpin:service" = "cache", "gistpin:sub-service" = "redis" } + analytics = { "gistpin:service" = "analytics", "gistpin:sub-service" = "data-pipeline" } + cdn = { "gistpin:service" = "cdn", "gistpin:sub-service" = "cloudfront" } + search = { "gistpin:service" = "search", "gistpin:sub-service" = "elasticsearch" } + queue = { "gistpin:service" = "queue", "gistpin:sub-service" = "sqs" } + monitoring = { "gistpin:service" = "monitoring", "gistpin:sub-service" = "observability" } + networking = { "gistpin:service" = "networking", "gistpin:sub-service" = "vpc" } + security = { "gistpin:service" = "security", "gistpin:sub-service" = "iam" } + ci_cd = { "gistpin:service" = "ci-cd", "gistpin:sub-service" = "github-actions" } + } +} diff --git a/infrastructure/terraform/tagging-taxonomy.tf b/infrastructure/terraform/tagging-taxonomy.tf new file mode 100644 index 00000000..8518548e --- /dev/null +++ b/infrastructure/terraform/tagging-taxonomy.tf @@ -0,0 +1,131 @@ +################################################################################ +# Tagging Taxonomy – comprehensive tag schema for GistPin +################################################################################ + +variable "tag_taxonomy_version" { + description = "Version of the tagging taxonomy" + type = string + default = "1.0" +} + +locals { + # ── Automation tags ────────────────────────────────────────────────── + automation_tags = { + "gistpin:environment" = var.environment + "gistpin:managed-by" = "terraform" + "gistpin:provisioner" = "terraform-cloud" + "gistpin:terraform-workspace" = terraform.workspace + } + + # ── Cost allocation tags ───────────────────────────────────────────── + cost_tags = { + "gistpin:cost-center" = var.cost_center + "gistpin:project" = var.project_name + "gistpin:service" = "" + "gistpin:sub-service" = "" + "gistpin:billing-code" = "" + } + + # ── Ownership tags ─────────────────────────────────────────────────── + ownership_tags = { + "gistpin:owner" = var.owner + "gistpin:team" = "" + "gistpin:contact-email" = "" + "gistpin:slack-channel" = "" + } + + # ── Operational tags ───────────────────────────────────────────────── + operational_tags = { + "gistpin:tier" = "" + "gistpin:criticality" = "" + "gistpin:backup" = "" + "gistpin:retention" = "" + "gistpin:maintenance-window" = "" + } + + # ── Security & compliance tags ─────────────────────────────────────── + security_tags = { + "gistpin:data-classification" = "" + "gistpin:compliance" = "" + "gistpin:encryption-required" = "" + "gistpin:audit-enabled" = "" + } + + # ── Lifecycle tags ─────────────────────────────────────────────────── + lifecycle_tags = { + "gistpin:created-by" = "" + "gistpin:created-date" = "" + "gistpin:decommission-date" = "" + "gistpin:ttl" = "" + } + + # Combined taxonomy + tagging_taxonomy = merge( + local.automation_tags, + local.cost_tags, + local.ownership_tags, + local.operational_tags, + local.security_tags, + local.lifecycle_tags, + ) + + # Required tag keys that must be present on all resources + required_tag_keys = keys(local.tagging_taxonomy) +} + +# ── Sentinel policy to enforce tagging taxonomy ────────────────────────── +resource "aws_s3_object" "tagging_sentinel_policy" { + bucket = aws_s3_bucket.sentinel_policies.id + key = "enforce-tagging-taxonomy.sentinel" + content = templatefile("${path.module}/templates/enforce-tagging-taxonomy.sentinel.tftpl", { + required_tags = jsonencode(local.required_tag_keys) + taxonomy_ver = var.tag_taxonomy_version + }) + etag = filemd5("${path.module}/templates/enforce-tagging-taxonomy.sentinel.tftpl") +} + +# ── AWS Config rule to check required tags ────────────────────────────── +resource "aws_config_config_rule" "tagging_taxonomy" { + name = "${var.project_name}-tagging-taxonomy" + description = "Ensures all resources comply with the GistPin tagging taxonomy v${var.tag_taxonomy_version}" + + source { + owner = "AWS" + source_identifier = "REQUIRED_TAGS" + } + + input_parameters = jsonencode({ + tag1Key = "gistpin:environment" + tag2Key = "gistpin:cost-center" + tag3Key = "gistpin:project" + tag4Key = "gistpin:owner" + tag5Key = "gistpin:managed-by" + }) +} + +# ── Report non-compliant resources ────────────────────────────────────── +resource "aws_config_config_rule" "tagging_taxonomy_automation" { + name = "${var.project_name}-tagging-automation" + description = "Ensures automation tags are present" + + source { + owner = "AWS" + source_identifier = "REQUIRED_TAGS" + } + + input_parameters = jsonencode({ + tag1Key = "gistpin:environment" + tag2Key = "gistpin:managed-by" + tag3Key = "gistpin:provisioner" + }) +} + +output "tagging_taxonomy_keys" { + description = "All tagging taxonomy keys" + value = local.required_tag_keys +} + +output "tagging_taxonomy_version" { + description = "Current taxonomy version" + value = var.tag_taxonomy_version +}