diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml new file mode 100644 index 0000000..b6ef383 --- /dev/null +++ b/.github/workflows/pr-checks.yml @@ -0,0 +1,95 @@ +# ABOUTME: Runs safe PR validation checks for Rust formatting, tests, and integration coverage. +# ABOUTME: Avoids exposing repository credentials to code from pull request branches. +name: PR checks + +on: + pull_request: + +permissions: + contents: read + +jobs: + cargo-fmt: + name: cargo fmt --all --check + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - run: rustup component add rustfmt + - run: cargo fmt --all --check + + cargo-test: + name: cargo test + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 + - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + cache-on-failure: "true" + - run: cargo test --workspace + + integration-tests: + name: integration tests + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd + - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f + with: + context: . + file: ./Dockerfile + tags: pkgly:test + load: true + cache-from: type=gha,scope=pkgly + cache-to: type=gha,scope=pkgly,mode=max + - uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f + with: + context: ./tests/docker + file: ./tests/docker/Dockerfile.test-runner + tags: pkgly-test-runner:ci + load: true + cache-from: type=gha,scope=test-runner + cache-to: type=gha,scope=test-runner,mode=max + - run: ./tests/run_integration_tests.sh + env: + PKGLY_INTEGRATION_COMPOSE_OVERRIDE: ${{ github.workspace }}/tests/docker/docker-compose.ci-cache.yml + + chart-changes: + name: detect chart changes + runs-on: ubuntu-24.04 + outputs: + changed: ${{ steps.changes.outputs.changed }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + persist-credentials: false + - id: changes + run: | + set +e + git diff --quiet "${{ github.event.pull_request.base.sha }}" "${{ github.event.pull_request.head.sha }}" -- charts/ + status=$? + set -e + + case "$status" in + 0) echo "changed=false" >> "$GITHUB_OUTPUT" ;; + 1) echo "changed=true" >> "$GITHUB_OUTPUT" ;; + *) exit "$status" ;; + esac + + helm-lint: + name: helm lint and unit + needs: chart-changes + if: needs.chart-changes.outputs.changed == 'true' + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + with: + persist-credentials: false + - run: ./charts/pkgly/tests/unit.sh diff --git a/charts/pkgly/Chart.lock b/charts/pkgly/Chart.lock index 727c95a..87f248a 100644 --- a/charts/pkgly/Chart.lock +++ b/charts/pkgly/Chart.lock @@ -2,8 +2,5 @@ dependencies: - name: postgresql repository: https://charts.bitnami.com/bitnami version: 18.1.8 -- name: jaeger - repository: https://jaegertracing.github.io/helm-charts - version: 3.4.1 -digest: sha256:ba00edc6e27519de3e596b6c6b46a3d9855383b344d7537d5620a36573189c88 -generated: "2025-11-09T10:43:52.239090332+01:00" +digest: sha256:18ae170b87964c4d42c923aace1bdcce6bc990f55bf62dd4ff51fc1cc966a579 +generated: "2026-06-14T16:40:59.09543+02:00" diff --git a/charts/pkgly/Chart.yaml b/charts/pkgly/Chart.yaml index 6bb3a1c..2df4a1a 100644 --- a/charts/pkgly/Chart.yaml +++ b/charts/pkgly/Chart.yaml @@ -1,16 +1,12 @@ # ABOUTME: Helm chart metadata for Pkgly - Universal artifact repository. -# ABOUTME: Defines chart version, dependencies (PostgreSQL, Jaeger), and metadata. +# ABOUTME: Defines chart version, optional PostgreSQL dependency, and metadata. apiVersion: v2 -appVersion: 3.0.0-BETA +appVersion: 1.0.0-BETA dependencies: - condition: postgresql.enabled name: postgresql repository: https://charts.bitnami.com/bitnami version: 18.1.8 -- condition: jaeger.enabled - name: jaeger - repository: https://jaegertracing.github.io/helm-charts - version: 3.4.1 description: Pkgly - Universal artifact repository supporting various packages home: https://github.com/kshcherban/pkgly keywords: @@ -29,4 +25,4 @@ name: pkgly sources: - https://github.com/kshcherban/pkgly type: application -version: 0.4.0 +version: 0.5.0 diff --git a/charts/pkgly/README.md b/charts/pkgly/README.md index 8f6f051..4286271 100644 --- a/charts/pkgly/README.md +++ b/charts/pkgly/README.md @@ -7,7 +7,7 @@ This Helm chart deploys Pkgly, a universal artifact repository supporting Docker - **Multi-protocol support**: HTTP, OCI, and native package manager protocols - **Repository types**: Docker, Go proxy, Maven, NPM, PHP Composer, Python PyPI, Helm charts - **Database**: PostgreSQL (included as dependency) -- **Tracing**: Jaeger integration (optional) +- **Tracing**: OpenTelemetry export to an operator-managed collector - **Storage**: Local filesystem (with PVC support) - **Security**: Security contexts, resource limits, health checks - **Ingress**: Nginx ingress support with large file upload handling @@ -25,11 +25,11 @@ Add the repository and install the chart: ```bash # Install dependencies helm repo add bitnami https://charts.bitnami.com/bitnami -helm repo add jaegertracing https://jaegertracing.github.io/helm-charts helm repo update # Install the chart -helm install my-pkgly ./pkgly-chart +helm dependency build ./charts/pkgly +helm install my-pkgly ./charts/pkgly ``` ## Uninstalling the Chart @@ -45,29 +45,26 @@ See `values.yaml` for configuration options. The following table shows the confi | Parameter | Description | Default | |-----------|-------------|---------| | `replicaCount` | Number of Pkgly replicas | `1` | -| `image.repository` | Container image repository | `ghcr.io/sudoers/pkgly` | +| `image.repository` | Container image repository | `ghcr.io/pkgly/pkgly` | | `image.tag` | Container image tag | `latest` | | `service.type` | Kubernetes service type | `ClusterIP` | | `service.port` | Service port | `6742` | | `ingress.enabled` | Enable ingress | `false` | | `postgresql.enabled` | Deploy PostgreSQL | `true` | -| `jaeger.enabled` | Deploy Jaeger for tracing | `false` | +| `opentelemetry.endpoint` | Operator-managed OTLP collector endpoint | `http://otel-collector.observability.svc:4317` | | `persistence.enabled` | Enable persistent storage | `true` | | `persistence.size` | Storage size | `10Gi` | -| `rustLog` | Rust log level | `info` | | `env` | Custom environment variables with secret support | `[]` | -| `externalDatabase.secretRef.enabled` | Use secret for external database URL | `false` | +| `externalDatabase.secretRef.enabled` | Use a Secret for the external database password | `false` | -### Development Mode +### OpenTelemetry -For development with Jaeger tracing enabled: +Jaeger is not installed by this chart. Deploy an OpenTelemetry collector or Jaeger separately, then configure its OTLP endpoint: ```yaml -tracing: +opentelemetry: enabled: true -jaeger: - enabled: true -rustLog: "debug" + endpoint: "http://otel-collector.observability.svc:4317" ``` ### Production Mode @@ -94,35 +91,42 @@ postgresql: ## Persistence -The chart uses a PersistentVolumeClaim for storing repository data. The PVC is created by default and uses the default storage class. +The chart uses a PersistentVolumeClaim for repository data, sessions, staging files, and extracted frontend assets. The PVC is created by default and uses the default storage class. Helm upgrades and pod restarts reuse the same claim. + +The bundled PostgreSQL dependency also enables its own PersistentVolumeClaim by default. + +The liveness probe calls `GET /health`. Pkgly exposes this endpoint after application startup and database initialization complete. ## Database By default, the chart deploys PostgreSQL as a dependency. You can disable it and use an external database: -### External Database with Secret (Recommended) +### External Database with Secret -For production environments, use a Kubernetes secret to store the database URL: +Disable the bundled dependency and provide connection fields. Use a Kubernetes Secret for the password: ```yaml postgresql: enabled: false externalDatabase: + host: postgres.database.svc + port: 5432 + user: pkgly + database: pkgly secretRef: enabled: true name: database-secret - key: url - optional: false + passwordKey: password ``` Create the secret: ```bash kubectl create secret generic database-secret \ - --from-literal=url="postgresql://user:password@host:5432/database" + --from-literal=password="replace-me" ``` -### External Database with Direct URL +### External Database with Direct Password For development environments: @@ -130,7 +134,11 @@ For development environments: postgresql: enabled: false externalDatabase: - url: "postgresql://user:password@host:5432/database" + host: postgres.database.svc + port: 5432 + user: pkgly + password: pkgly + database: pkgly secretRef: enabled: false ``` @@ -170,7 +178,6 @@ env: - **Web UI**: `http://..svc.cluster.local:6742` - **API**: Same URL, with appropriate endpoints -- **Default credentials**: `admin` / `admin123` (change `adminPassword` in production) ## Repository URLs @@ -188,6 +195,16 @@ Once deployed, repositories are accessible at: helm upgrade my-pkgly ./pkgly-chart ``` +## Testing + +The chart test suite expects a local kind cluster named `kind`, the default namespace, and a local `pkgly:test` image: + +```bash +./charts/pkgly/tests/run.sh +``` + +Override the image or cluster with `PKGLY_TEST_IMAGE_REPOSITORY`, `PKGLY_TEST_IMAGE_TAG`, or `PKGLY_KIND_CLUSTER`. + ## Contributing -This chart is part of the Pkgly project. Please submit issues and PRs to the main repository. \ No newline at end of file +This chart is part of the Pkgly project. Please submit issues and PRs to the main repository. diff --git a/charts/pkgly/templates/configmap.yaml b/charts/pkgly/templates/configmap.yaml index 8cb6326..f935ab9 100644 --- a/charts/pkgly/templates/configmap.yaml +++ b/charts/pkgly/templates/configmap.yaml @@ -14,24 +14,10 @@ data: # ========================================================================== # Database Configuration # ========================================================================== + {{- if and (not .Values.postgresql.enabled) (not .Values.externalDatabase.secretRef.enabled) }} [database] - {{- if .Values.postgresql.enabled }} - user = {{ .Values.postgresql.auth.username | quote }} - {{- if .Values.postgresql.auth.existingSecret }} - password = "${DB_PASSWORD}" - {{- else }} - password = {{ .Values.postgresql.auth.password | quote }} - {{- end }} - database = {{ .Values.postgresql.auth.database | quote }} - host = "{{ include "pkgly-chart.fullname" . }}-postgresql" - port = 5432 - {{- else }} user = {{ .Values.externalDatabase.user | quote }} - {{- if .Values.externalDatabase.secretRef.enabled }} - password = "${DB_PASSWORD}" - {{- else }} password = {{ .Values.externalDatabase.password | quote }} - {{- end }} database = {{ .Values.externalDatabase.database | quote }} host = {{ .Values.externalDatabase.host | quote }} port = {{ .Values.externalDatabase.port }} @@ -43,9 +29,7 @@ data: [web_server] bind_address = "0.0.0.0:{{ .Values.service.port }}" open_api_routes = {{ .Values.webServer.openApiRoutes }} - - [web_server.max_upload] - max_size = {{ .Values.webServer.maxUpload | quote }} + max_upload = {{ .Values.webServer.maxUpload | quote }} # ========================================================================== # Session Configuration @@ -58,13 +42,14 @@ data: # ========================================================================== # Site Configuration # ========================================================================== - {{- if .Values.site.appUrl }} [site] - app_url = {{ .Values.site.appUrl | quote }} + {{- with .Values.site.appUrl }} + app_url = {{ . | quote }} + {{- end }} name = {{ .Values.site.name | quote }} description = {{ .Values.site.description | quote }} is_https = {{ .Values.site.isHttps }} - {{- end }} + frontend_path = {{ .Values.site.frontendPath | quote }} # ========================================================================== # Security Configuration @@ -95,10 +80,8 @@ data: # Staging Configuration # ========================================================================== [staging] - enabled = {{ .Values.staging.enabled }} - max_size = {{ .Values.staging.maxSize | quote }} - cleanup_interval = {{ .Values.staging.cleanupInterval }} - stale_timeout = {{ .Values.staging.staleTimeout }} + staging_dir = {{ .Values.staging.stagingDir | quote }} + time_till_cleanup = {{ .Values.staging.timeTillCleanup }} # ========================================================================== # S3 Storage Configuration @@ -134,11 +117,7 @@ data: [opentelemetry] enabled = true protocol = {{ .Values.opentelemetry.protocol | quote }} - {{- if .Values.jaeger.enabled }} - endpoint = "http://{{ include "pkgly-chart.fullname" . }}-jaeger-collector:4317" - {{- else }} endpoint = {{ .Values.opentelemetry.endpoint | quote }} - {{- end }} traces = {{ .Values.opentelemetry.traces }} logs = {{ .Values.opentelemetry.logs }} diff --git a/charts/pkgly/templates/deployment.yaml b/charts/pkgly/templates/deployment.yaml index 3906e69..6ffa6b3 100644 --- a/charts/pkgly/templates/deployment.yaml +++ b/charts/pkgly/templates/deployment.yaml @@ -8,6 +8,8 @@ metadata: {{- include "pkgly-chart.labels" . | nindent 4 }} spec: replicas: {{ .Values.replicaCount }} + strategy: + type: Recreate selector: matchLabels: {{- include "pkgly-chart.selectorLabels" . | nindent 6 }} @@ -46,19 +48,34 @@ spec: resources: {{- toYaml .Values.resources | nindent 12 }} env: - # Database password from secret (when using existingSecret or external DB secretRef) - {{- if and .Values.postgresql.enabled .Values.postgresql.auth.existingSecret }} - - name: DB_PASSWORD + {{- if .Values.postgresql.enabled }} + - name: PKGLY_DATABASE_USER + value: {{ .Values.postgresql.auth.username | quote }} + - name: PKGLY_DATABASE_PASSWORD valueFrom: secretKeyRef: - name: {{ .Values.postgresql.auth.existingSecret }} + name: {{ default (printf "%s-postgresql" (include "pkgly-chart.fullname" .)) .Values.postgresql.auth.existingSecret }} key: password + - name: PKGLY_DATABASE_DATABASE + value: {{ .Values.postgresql.auth.database | quote }} + - name: PKGLY_DATABASE_HOST + value: "{{ include "pkgly-chart.fullname" . }}-postgresql" + - name: PKGLY_DATABASE_PORT + value: "5432" {{- else if and (not .Values.postgresql.enabled) .Values.externalDatabase.secretRef.enabled }} - - name: DB_PASSWORD + - name: PKGLY_DATABASE_USER + value: {{ .Values.externalDatabase.user | quote }} + - name: PKGLY_DATABASE_PASSWORD valueFrom: secretKeyRef: name: {{ .Values.externalDatabase.secretRef.name }} key: {{ .Values.externalDatabase.secretRef.passwordKey }} + - name: PKGLY_DATABASE_DATABASE + value: {{ .Values.externalDatabase.database | quote }} + - name: PKGLY_DATABASE_HOST + value: {{ .Values.externalDatabase.host | quote }} + - name: PKGLY_DATABASE_PORT + value: {{ .Values.externalDatabase.port | quote }} {{- end }} # S3 credentials from secret (when using S3 with secretRef) {{- if and .Values.s3Storage.enabled .Values.s3Storage.secretRef.enabled (not .Values.s3Storage.useIamRole) }} diff --git a/charts/pkgly/templates/serviceaccount.yaml b/charts/pkgly/templates/serviceaccount.yaml new file mode 100644 index 0000000..c1637d8 --- /dev/null +++ b/charts/pkgly/templates/serviceaccount.yaml @@ -0,0 +1,15 @@ +# ABOUTME: Kubernetes ServiceAccount for the Pkgly pod. +# ABOUTME: Only created when serviceAccount.create is true; honors name/annotations/automount. +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "pkgly-chart.serviceAccountName" . }} + labels: + {{- include "pkgly-chart.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} +{{- end }} diff --git a/charts/pkgly/tests/e2e.sh b/charts/pkgly/tests/e2e.sh new file mode 100755 index 0000000..e430697 --- /dev/null +++ b/charts/pkgly/tests/e2e.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# ABOUTME: Verifies Pkgly and PostgreSQL data survive pod restarts and Helm redeploys. +# ABOUTME: Exercises real PVC-backed storage in local kind default namespace. +set -euo pipefail + +chart_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +release="${PKGLY_HELM_RELEASE:-pkgly-chart-test}" +timeout="${PKGLY_TEST_TIMEOUT:-5m}" +marker="pkgly-persistence-$(date +%s)" +app_selector="app.kubernetes.io/instance=$release,app.kubernetes.io/name=pkgly" +postgres_selector="app.kubernetes.io/instance=$release,app.kubernetes.io/name=postgresql" + +app_pod() { + kubectl get pod --namespace default --selector "$app_selector" \ + -o jsonpath='{.items[0].metadata.name}' +} + +postgres_pod() { + kubectl get pod --namespace default --selector "$postgres_selector" \ + -o jsonpath='{.items[0].metadata.name}' +} + +wait_for_workloads() { + kubectl rollout status "deployment/$release" --namespace default --timeout="$timeout" + kubectl rollout status "statefulset/$release-postgresql" --namespace default --timeout="$timeout" +} + +kubectl exec --namespace default "$(app_pod)" -- \ + sh -c "printf '%s' '$marker' > /data/helm-persistence-marker" +kubectl exec --namespace default "$(postgres_pod)" -- \ + env PGPASSWORD=pkgly psql -U pkgly -d pkgly -v ON_ERROR_STOP=1 \ + -c "CREATE TABLE IF NOT EXISTS helm_persistence_test (value text PRIMARY KEY);" \ + -c "INSERT INTO helm_persistence_test (value) VALUES ('$marker');" + +kubectl delete pod --namespace default "$(app_pod)" "$(postgres_pod)" --wait=false +wait_for_workloads + +test "$(kubectl exec --namespace default "$(app_pod)" -- cat /data/helm-persistence-marker)" = "$marker" +kubectl exec --namespace default "$(postgres_pod)" -- \ + env PGPASSWORD=pkgly psql -U pkgly -d pkgly -tAc \ + "SELECT value FROM helm_persistence_test WHERE value = '$marker';" | + grep -qx "$marker" + +helm upgrade "$release" "$chart_dir" --namespace default --reuse-values --wait --timeout "$timeout" +wait_for_workloads + +test "$(kubectl exec --namespace default "$(app_pod)" -- cat /data/helm-persistence-marker)" = "$marker" +kubectl exec --namespace default "$(postgres_pod)" -- \ + env PGPASSWORD=pkgly psql -U pkgly -d pkgly -tAc \ + "SELECT value FROM helm_persistence_test WHERE value = '$marker';" | + grep -qx "$marker" + +echo "Helm chart end-to-end persistence tests passed" diff --git a/charts/pkgly/tests/integration.sh b/charts/pkgly/tests/integration.sh new file mode 100755 index 0000000..d4f7782 --- /dev/null +++ b/charts/pkgly/tests/integration.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# ABOUTME: Installs Pkgly into local kind and verifies Kubernetes workloads become ready. +# ABOUTME: Uses default namespace and chart-bundled PostgreSQL dependency. +set -euo pipefail + +chart_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +release="${PKGLY_HELM_RELEASE:-pkgly-chart-test}" +external_release="${release}-external" +external_database_secret="${external_release}-database" +image_repository="${PKGLY_TEST_IMAGE_REPOSITORY:-pkgly}" +image_tag="${PKGLY_TEST_IMAGE_TAG:-test}" +timeout="${PKGLY_TEST_TIMEOUT:-5m}" +trap 'helm uninstall "$external_release" --namespace default --wait >/dev/null 2>&1 || true; kubectl delete secret "$external_database_secret" --namespace default --ignore-not-found >/dev/null 2>&1' EXIT + +helm upgrade --install "$release" "$chart_dir" \ + --namespace default \ + --set image.repository="$image_repository" \ + --set image.tag="$image_tag" \ + --set image.pullPolicy=IfNotPresent \ + --wait \ + --timeout "$timeout" + +kubectl rollout status "deployment/$release" --namespace default --timeout="$timeout" +kubectl rollout status "statefulset/$release-postgresql" --namespace default --timeout="$timeout" +kubectl get pod --namespace default \ + --selector "app.kubernetes.io/instance=$release" \ + -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.phase}{"\n"}{end}' | + awk '$2 != "Running" { exit 1 }' + +kubectl delete secret "$external_database_secret" --namespace default --ignore-not-found +kubectl create secret generic "$external_database_secret" \ + --namespace default \ + --from-literal=password=pkgly + +helm upgrade --install "$external_release" "$chart_dir" \ + --namespace default \ + --set image.repository="$image_repository" \ + --set image.tag="$image_tag" \ + --set image.pullPolicy=IfNotPresent \ + --set persistence.enabled=false \ + --set postgresql.enabled=false \ + --set externalDatabase.host="$release-postgresql" \ + --set externalDatabase.user=pkgly \ + --set externalDatabase.database=pkgly \ + --set externalDatabase.secretRef.enabled=true \ + --set externalDatabase.secretRef.name="$external_database_secret" \ + --wait \ + --timeout "$timeout" + +kubectl rollout status "deployment/$external_release" --namespace default --timeout="$timeout" +test "$(kubectl get statefulset --namespace default \ + --selector "app.kubernetes.io/instance=$external_release" \ + --no-headers 2>/dev/null | wc -l | tr -d ' ')" = "0" + +echo "Helm chart integration tests passed" diff --git a/charts/pkgly/tests/run.sh b/charts/pkgly/tests/run.sh new file mode 100755 index 0000000..641844c --- /dev/null +++ b/charts/pkgly/tests/run.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# ABOUTME: Runs Pkgly Helm chart unit, integration, and end-to-end tests. +# ABOUTME: Loads the configured local image into kind before cluster tests. +set -euo pipefail + +tests_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +image_repository="${PKGLY_TEST_IMAGE_REPOSITORY:-pkgly}" +image_tag="${PKGLY_TEST_IMAGE_TAG:-test}" +kind_cluster="${PKGLY_KIND_CLUSTER:-kind}" + +"$tests_dir/unit.sh" +kind load docker-image "$image_repository:$image_tag" --name "$kind_cluster" +"$tests_dir/integration.sh" +"$tests_dir/e2e.sh" diff --git a/charts/pkgly/tests/unit.sh b/charts/pkgly/tests/unit.sh new file mode 100755 index 0000000..26a12ce --- /dev/null +++ b/charts/pkgly/tests/unit.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# ABOUTME: Runs static and rendered unit checks for the Pkgly Helm chart. +# ABOUTME: Verifies optional PostgreSQL behavior and external OpenTelemetry configuration. +set -euo pipefail + +chart_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +rendered_default="$(mktemp)" +rendered_external="$(mktemp)" +rendered_external_secret="$(mktemp)" +trap 'rm -f "$rendered_default" "$rendered_external" "$rendered_external_secret"' EXIT + +helm repo add bitnami https://charts.bitnami.com/bitnami --force-update +helm dependency build --skip-refresh "$chart_dir" +helm lint "$chart_dir" +helm template pkgly "$chart_dir" --namespace default >"$rendered_default" +helm template pkgly "$chart_dir" \ + --namespace default \ + --set postgresql.enabled=false \ + --set externalDatabase.host=postgres.example.internal \ + --set externalDatabase.user=pkgly \ + --set externalDatabase.password=secret \ + --set externalDatabase.database=pkgly \ + --set opentelemetry.enabled=true \ + --set opentelemetry.endpoint=http://otel-collector.observability.svc:4317 \ + >"$rendered_external" +helm template pkgly "$chart_dir" \ + --namespace default \ + --set postgresql.enabled=false \ + --set externalDatabase.host=postgres.example.internal \ + --set externalDatabase.user=pkgly \ + --set externalDatabase.database=pkgly \ + --set externalDatabase.secretRef.enabled=true \ + --set externalDatabase.secretRef.name=database-secret \ + >"$rendered_external_secret" + +if grep -Eq '(^|[[:space:]])jaeger([:.]|$)' "$chart_dir/Chart.yaml" "$chart_dir/Chart.lock"; then + echo "Jaeger must not be a chart dependency" >&2 + exit 1 +fi + +if grep -q 'name: pkgly-postgresql' "$rendered_external"; then + echo "PostgreSQL resources rendered while postgresql.enabled=false" >&2 + exit 1 +fi + +grep -q 'host = "postgres.example.internal"' "$rendered_external" +grep -q 'endpoint = "http://otel-collector.observability.svc:4317"' "$rendered_external" +grep -q 'name: PKGLY_DATABASE_PASSWORD' "$rendered_external_secret" +grep -q 'name: PKGLY_DATABASE_HOST' "$rendered_external_secret" +if grep -q '\${DB_PASSWORD}' "$rendered_external_secret"; then + echo "Pkgly does not interpolate environment variables inside TOML values" >&2 + exit 1 +fi +grep -q 'kind: StatefulSet' "$rendered_default" +grep -q 'name: pkgly-postgresql' "$rendered_default" +grep -q 'claimName: pkgly-data' "$rendered_default" +grep -q 'name: PKGLY_DATABASE_PASSWORD' "$rendered_default" +grep -A1 '^ strategy:' "$rendered_default" | grep -q 'type: Recreate' +grep -A4 'livenessProbe:' "$rendered_default" | grep -q 'path: /health' +grep -A4 'livenessProbe:' "$rendered_default" | grep -q 'port: http' +grep -A5 'readinessProbe:' "$rendered_default" | grep -q 'tcpSocket:' + +echo "Helm chart unit tests passed" diff --git a/charts/pkgly/values.yaml b/charts/pkgly/values.yaml index 93734e7..1f93a2e 100644 --- a/charts/pkgly/values.yaml +++ b/charts/pkgly/values.yaml @@ -1,5 +1,5 @@ -# Default values for pkgly-chart. -# This is a YAML-formatted file. +# ABOUTME: Default values for configuring Pkgly Kubernetes resources and application settings. +# ABOUTME: Enables persistent local storage and bundled PostgreSQL unless explicitly disabled. # Declare variables to be passed into your templates. # Pkgly deployment settings @@ -88,8 +88,7 @@ livenessProbe: failureThreshold: 3 readinessProbe: - httpGet: - path: /health + tcpSocket: port: http initialDelaySeconds: 5 periodSeconds: 5 @@ -106,11 +105,12 @@ site: name: "Pkgly" description: "An Open Source artifact manager" isHttps: true + frontendPath: "/data/frontend" # Must be writable by the container user (under the /data PVC) # Web server configuration webServer: openApiRoutes: true - maxUpload: "100MB" + maxUpload: "100MiB" # Session configuration sessions: @@ -140,16 +140,14 @@ oauth2: # Staging configuration staging: - enabled: true - maxSize: "1GB" - cleanupInterval: 3600 # seconds - staleTimeout: 86400 # seconds (24 hours) + stagingDir: "/data/staging" + timeTillCleanup: 3600 # seconds (1 hour) # OpenTelemetry configuration opentelemetry: enabled: false protocol: "GRPC" # Options: GRPC, HttpBinary, HttpJson - endpoint: "http://localhost:4317" + endpoint: "http://otel-collector.observability.svc:4317" traces: true logs: false config: @@ -212,7 +210,7 @@ postgresql: enabled: true auth: username: pkgly - password: "pkgly" # Only used when creating the database, not stored in config files + password: "pkgly" # Stored in the PostgreSQL Secret and consumed by Pkgly from that Secret database: pkgly existingSecret: "" primary: @@ -229,35 +227,16 @@ postgresql: # External database configuration (when postgresql.enabled = false) externalDatabase: - # Option 1: Individual fields (recommended) host: "" port: 5432 user: "" password: "" database: "" - # Option 2: Reference K8s secret for password (more secure) secretRef: enabled: false name: "database-secret" passwordKey: "password" -# Jaeger dependency configuration -jaeger: - enabled: false - provisionDataStore: - cassandra: false - elasticsearch: false - kafka: false - allInOne: - enabled: true - image: - repository: jaegertracing/all-in-one - tag: latest - options: - log-level: info - storage: - type: badger - # Node selection nodeSelector: {} diff --git a/crates/pkgly-cli/tests/command_tests.rs b/crates/pkgly-cli/tests/command_tests.rs index c42bb39..455b3ce 100644 --- a/crates/pkgly-cli/tests/command_tests.rs +++ b/crates/pkgly-cli/tests/command_tests.rs @@ -106,13 +106,22 @@ fn parse_content_length(headers: &[u8]) -> usize { fn json_response(status: &str, body: &str) -> String { format!( - "HTTP/1.1 {status}\r\nContent-Length: {}\r\nContent-Type: application/json\r\n\r\n{body}", + "HTTP/1.1 {status}\r\nContent-Length: {}\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n{body}", body.len() ) } fn empty_response(status: &str) -> String { - format!("HTTP/1.1 {status}\r\nContent-Length: 0\r\n\r\n") + format!("HTTP/1.1 {status}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") +} + +fn login_response() -> String { + "HTTP/1.1 200 OK\r\nSet-Cookie: session=session123; Path=/\r\nContent-Length: 2\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n{}".to_string() +} + +#[test] +fn login_response_closes_connection() { + assert!(login_response().contains("\r\nConnection: close\r\n")); } fn package_entry_json(package: &str, version: &str) -> String { @@ -205,11 +214,8 @@ async fn repo_list_command_uses_configured_server_and_token() { #[tokio::test] async fn auth_login_stores_created_token_without_password() { let token_body = "{\"id\":1,\"token\":\"created-token\",\"expires_at\":null}"; - let server = MockServer::start(vec![ - "HTTP/1.1 200 OK\r\nSet-Cookie: session=session123; Path=/\r\nContent-Length: 2\r\nContent-Type: application/json\r\n\r\n{}".to_string(), - json_response("200 OK", token_body), - ]) - .unwrap_or_else(|err| panic!("mock server failed: {err}")); + let server = MockServer::start(vec![login_response(), json_response("200 OK", token_body)]) + .unwrap_or_else(|err| panic!("mock server failed: {err}")); let temp = tempfile::tempdir().unwrap_or_else(|err| panic!("tempdir failed: {err}")); let config_path = temp.path().join("config.toml"); let mut config = ConfigFile::default(); @@ -263,11 +269,8 @@ async fn auth_login_stores_created_token_without_password() { #[tokio::test] async fn auth_login_stores_base_url_from_flag_in_new_profile() { let token_body = "{\"id\":1,\"token\":\"created-token\",\"expires_at\":null}"; - let server = MockServer::start(vec![ - "HTTP/1.1 200 OK\r\nSet-Cookie: session=session123; Path=/\r\nContent-Length: 2\r\nContent-Type: application/json\r\n\r\n{}".to_string(), - json_response("200 OK", token_body), - ]) - .unwrap_or_else(|err| panic!("mock server failed: {err}")); + let server = MockServer::start(vec![login_response(), json_response("200 OK", token_body)]) + .unwrap_or_else(|err| panic!("mock server failed: {err}")); let temp = tempfile::tempdir().unwrap_or_else(|err| panic!("tempdir failed: {err}")); let config_path = temp.path().join("config.toml"); @@ -304,11 +307,8 @@ async fn auth_login_stores_base_url_from_flag_in_new_profile() { #[tokio::test] async fn auth_login_prompts_for_password_when_flag_is_absent() { let token_body = "{\"id\":1,\"token\":\"created-token\",\"expires_at\":null}"; - let server = MockServer::start(vec![ - "HTTP/1.1 200 OK\r\nSet-Cookie: session=session123; Path=/\r\nContent-Length: 2\r\nContent-Type: application/json\r\n\r\n{}".to_string(), - json_response("200 OK", token_body), - ]) - .unwrap_or_else(|err| panic!("mock server failed: {err}")); + let server = MockServer::start(vec![login_response(), json_response("200 OK", token_body)]) + .unwrap_or_else(|err| panic!("mock server failed: {err}")); let temp = tempfile::tempdir().unwrap_or_else(|err| panic!("tempdir failed: {err}")); let config_path = temp.path().join("config.toml"); diff --git a/crates/storage/src/local/mod.rs b/crates/storage/src/local/mod.rs index e33f693..399cadf 100644 --- a/crates/storage/src/local/mod.rs +++ b/crates/storage/src/local/mod.rs @@ -360,7 +360,7 @@ impl LocalStorageInner { }; let entry = entry; let path = entry.path(); - if path.is_file() && is_hidden_file(&path) { + if is_hidden_file(&path) { trace!(?path, "Skipping Meta File"); files_skipped += 1; // Check if file is a meta file diff --git a/crates/storage/src/local/stream.rs b/crates/storage/src/local/stream.rs index b1c237b..665349a 100644 --- a/crates/storage/src/local/stream.rs +++ b/crates/storage/src/local/stream.rs @@ -78,7 +78,7 @@ impl Stream for LocalDirectoryListStream { let path = entry.map_err(LocalStorageError::from)?; - if path.is_file() && is_hidden_file(&path) { + if is_hidden_file(&path) { // I am not really sure what to do with hidden files in a stream? // I am guessing if return Poll::Pending now it will just skip the file? trace!(?path, "Skipping Meta File"); diff --git a/crates/storage/src/local/tests.rs b/crates/storage/src/local/tests.rs index c1006d4..207b654 100644 --- a/crates/storage/src/local/tests.rs +++ b/crates/storage/src/local/tests.rs @@ -1,3 +1,5 @@ +// ABOUTME: Tests local storage filesystem behavior, locking, metadata, and repository cleanup. +// ABOUTME: Covers eager and streamed directory listings against transient internal metadata paths. #![allow(clippy::expect_used, clippy::panic, clippy::todo, clippy::unwrap_used)] use super::*; use crate::{ @@ -7,6 +9,7 @@ use crate::{ testing::storage::TestingStorage, }; use fs2::FileExt; +use futures::StreamExt; use nr_core::storage::StoragePath; use tempfile::tempdir; use tokio::time::{Duration, sleep}; @@ -340,3 +343,68 @@ async fn directory_entries_are_relative_and_sorted() -> anyhow::Result<()> { Ok(()) } + +#[cfg(unix)] +#[tokio::test] +async fn open_folder_ignores_disappeared_temporary_metadata_files() -> anyhow::Result<()> { + let temp = tempdir()?; + let storage = + ::create_storage_from_config(StorageConfig { + storage_config: StorageConfigInner::test_config(), + type_config: StorageTypeConfig::Local(LocalConfig { + path: temp.path().to_path_buf(), + }), + }) + .await?; + + let repository = Uuid::new_v4(); + let repository_root = temp.path().join(repository.to_string()); + std::fs::create_dir_all(&repository_root)?; + std::os::unix::fs::symlink( + repository_root.join("metadata-already-renamed"), + repository_root.join(".nr-meta.tmp-race.nr-meta"), + )?; + + let directory = storage + .open_file(repository, &StoragePath::default()) + .await? + .expect("repository directory should exist"); + + assert!(directory.directory().expect("directory").0.is_empty()); + Ok(()) +} + +#[cfg(unix)] +#[tokio::test] +async fn directory_stream_ignores_disappeared_temporary_metadata_files() -> anyhow::Result<()> { + let temp = tempdir()?; + let storage = + ::create_storage_from_config(StorageConfig { + storage_config: StorageConfigInner::test_config(), + type_config: StorageTypeConfig::Local(LocalConfig { + path: temp.path().to_path_buf(), + }), + }) + .await?; + + let repository = Uuid::new_v4(); + let repository_root = temp.path().join(repository.to_string()); + std::fs::create_dir_all(&repository_root)?; + std::os::unix::fs::symlink( + repository_root.join("metadata-already-renamed"), + repository_root.join(".nr-meta.tmp-race.nr-meta"), + )?; + + let mut stream = storage + .stream_directory(repository, &StoragePath::default()) + .await? + .expect("repository directory should exist"); + + let mut entries = 0; + while let Some(entry) = stream.next().await { + assert!(entry?.is_none()); + entries += 1; + } + assert_eq!(entries, 2); + Ok(()) +} diff --git a/docs/docs/quickstart.md b/docs/docs/quickstart.md index 4e2d5b8..24f7ba7 100644 --- a/docs/docs/quickstart.md +++ b/docs/docs/quickstart.md @@ -54,6 +54,8 @@ docker compose down -v Use this when you want a Kubernetes deployment quickly, with PostgreSQL managed by the chart. +The chart uses `GET /health` as the Pkgly liveness probe. The endpoint becomes available after Pkgly initializes and connects to PostgreSQL. + ### Prerequisites - Kubernetes cluster diff --git a/pkgly/src/app/routes.rs b/pkgly/src/app/routes.rs index 3968acd..74e2945 100644 --- a/pkgly/src/app/routes.rs +++ b/pkgly/src/app/routes.rs @@ -5,7 +5,7 @@ use axum::{ extract::{DefaultBodyLimit, Path, State}, response::{IntoResponse, Response}, }; -use http::{HeaderName, HeaderValue}; +use http::{HeaderName, HeaderValue, StatusCode}; use tower_http::set_header::SetResponseHeaderLayer; use tracing::info; @@ -20,6 +20,9 @@ use crate::{ use super::Pkgly; +#[cfg(test)] +mod tests; + const POWERED_BY_HEADER: HeaderName = HeaderName::from_static("x-powered-by"); const POWERED_BY_VALUE: HeaderValue = HeaderValue::from_static("Pkgly"); @@ -31,6 +34,7 @@ pub fn build_app_router(site: Pkgly, max_upload: MaxUpload, open_api_routes: boo info!("Docker V2 routes will be added at /v2"); let mut app = Router::new() + .route("/health", axum::routing::get(health)) // Docker Registry V2 API compatibility route // Handle both /v2 and /v2/ explicitly to work around Axum nesting trailing slash behavior .route( @@ -79,6 +83,10 @@ pub fn build_app_router(site: Pkgly, max_upload: MaxUpload, open_api_routes: boo .layer(body_limit) } +async fn health() -> StatusCode { + StatusCode::OK +} + async fn direct_repository_or_frontend_request( State(site): State, Path(request_path): Path, diff --git a/pkgly/src/app/routes/tests.rs b/pkgly/src/app/routes/tests.rs new file mode 100644 index 0000000..8053cc1 --- /dev/null +++ b/pkgly/src/app/routes/tests.rs @@ -0,0 +1,10 @@ +// ABOUTME: Verifies application-level routes that do not require authenticated state. +// ABOUTME: Covers the health response used by Kubernetes liveness probes. +use http::StatusCode; + +use super::health; + +#[tokio::test] +async fn health_returns_ok() { + assert_eq!(health().await, StatusCode::OK); +} diff --git a/tests/docker/docker-compose.ci-cache.yml b/tests/docker/docker-compose.ci-cache.yml new file mode 100644 index 0000000..7d37065 --- /dev/null +++ b/tests/docker/docker-compose.ci-cache.yml @@ -0,0 +1,5 @@ +# ABOUTME: CI-only Compose override that adds an explicit image tag for test-runner. +# ABOUTME: Allows docker/build-push-action to pre-build images that compose starts without rebuilding. +services: + test-runner: + image: pkgly-test-runner:ci diff --git a/tests/run_integration_tests.sh b/tests/run_integration_tests.sh index ec6c607..36d0138 100755 --- a/tests/run_integration_tests.sh +++ b/tests/run_integration_tests.sh @@ -10,6 +10,12 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DOCKER_DIR="${SCRIPT_DIR}/docker" INTEGRATION_DIR="${SCRIPT_DIR}/integration" +if [ -n "${PKGLY_INTEGRATION_COMPOSE_OVERRIDE:-}" ] && [ -f "$PKGLY_INTEGRATION_COMPOSE_OVERRIDE" ]; then + COMPOSE_CMD=(docker compose -f "${DOCKER_DIR}/docker-compose.test.yml" -f "$PKGLY_INTEGRATION_COMPOSE_OVERRIDE") +else + COMPOSE_CMD=(docker compose -f "${DOCKER_DIR}/docker-compose.test.yml") +fi + # Colors RED='\033[0;31m' GREEN='\033[0;32m' @@ -149,7 +155,7 @@ print_header # Clean up if requested if [ $CLEAN -eq 1 ]; then print_color "$YELLOW" "Cleaning up test environment..." - docker compose -f "${DOCKER_DIR}/docker-compose.test.yml" down -v > /dev/null 2>&1 || true + "${COMPOSE_CMD[@]}" down -v > /dev/null 2>&1 || true print_color "$GREEN" "✓ Cleanup complete" echo "" fi @@ -157,13 +163,13 @@ fi # Build images if requested if [ $BUILD -eq 1 ]; then print_color "$YELLOW" "Building Docker images..." - docker compose -f "${DOCKER_DIR}/docker-compose.test.yml" build + "${COMPOSE_CMD[@]}" build print_color "$GREEN" "✓ Build complete" echo "" fi REQUIRED_SERVICES=(postgres pkgly test-runner docker) -RUNNING_SERVICES=$(docker compose -f "${DOCKER_DIR}/docker-compose.test.yml" ps --status running --services 2>/dev/null || true) +RUNNING_SERVICES=$("${COMPOSE_CMD[@]}" ps --status running --services 2>/dev/null || true) ALL_REQUIRED_RUNNING=1 for svc in "${REQUIRED_SERVICES[@]}"; do if ! grep -qx "$svc" <<<"$RUNNING_SERVICES"; then @@ -186,7 +192,7 @@ for suite in "${TEST_SUITES[@]}"; do done if [ $REUSE_ENV -eq 1 ] && [ $NEEDS_ACCESS_LOG_VOLUME -eq 1 ]; then - if ! docker compose -f "${DOCKER_DIR}/docker-compose.test.yml" exec -T test-runner \ + if ! "${COMPOSE_CMD[@]}" exec -T test-runner \ test -d /pkgly-storage 2>/dev/null; then REUSE_ENV=0 fi @@ -198,7 +204,7 @@ if [ $REUSE_ENV -eq 1 ]; then else # Start test environment print_color "$YELLOW" "Starting test environment..." - docker compose -f "${DOCKER_DIR}/docker-compose.test.yml" up -d + "${COMPOSE_CMD[@]}" up -d # Wait for services to be healthy print_color "$YELLOW" "Waiting for services to be ready..." @@ -206,8 +212,8 @@ else ELAPSED=0 while [ $ELAPSED -lt $MAX_WAIT ]; do - if docker compose -f "${DOCKER_DIR}/docker-compose.test.yml" ps | grep -q "healthy"; then - POSTGRES_HEALTHY=$(docker compose -f "${DOCKER_DIR}/docker-compose.test.yml" ps postgres | grep -q "healthy" && echo 1 || echo 0) + if "${COMPOSE_CMD[@]}" ps | grep -q "healthy"; then + POSTGRES_HEALTHY=$("${COMPOSE_CMD[@]}" ps postgres | grep -q "healthy" && echo 1 || echo 0) if [ "$POSTGRES_HEALTHY" -eq 1 ]; then print_color "$GREEN" "✓ All services are healthy" break @@ -224,14 +230,14 @@ else if [ $ELAPSED -ge $MAX_WAIT ]; then print_color "$RED" "✗ Services did not become healthy within ${MAX_WAIT} seconds" - docker compose -f "${DOCKER_DIR}/docker-compose.test.yml" logs - docker compose -f "${DOCKER_DIR}/docker-compose.test.yml" down + "${COMPOSE_CMD[@]}" logs + "${COMPOSE_CMD[@]}" down exit 1 fi # Restart pkgly to load seeded repositories print_color "$YELLOW" "Restarting Pkgly to load seeded repositories..." - docker compose -f "${DOCKER_DIR}/docker-compose.test.yml" restart pkgly + "${COMPOSE_CMD[@]}" restart pkgly sleep 2 # Give it time to restart print_color "$GREEN" "✓ Pkgly restarted" @@ -261,7 +267,7 @@ for suite in "${TEST_SUITES[@]}"; do chmod +x "$TEST_SCRIPT" # Run test in test-runner container - if docker compose -f "${DOCKER_DIR}/docker-compose.test.yml" exec -T test-runner \ + if "${COMPOSE_CMD[@]}" exec -T test-runner \ bash "/tests/test_${suite}.sh"; then print_color "$GREEN" "✓ ${suite} tests PASSED" ((PASSED_SUITES+=1)) @@ -294,12 +300,12 @@ echo "" # Stop environment if requested if [ $STOP -eq 1 ] && [ $KEEP_RUNNING -eq 0 ]; then print_color "$YELLOW" "Stopping test environment..." - docker compose -f "${DOCKER_DIR}/docker-compose.test.yml" down + "${COMPOSE_CMD[@]}" down print_color "$GREEN" "✓ Environment stopped" else print_color "$YELLOW" "Test environment is still running" - echo " To view logs: docker compose -f ${DOCKER_DIR}/docker-compose.test.yml logs -f" - echo " To stop: docker compose -f ${DOCKER_DIR}/docker-compose.test.yml down" + echo " To view logs: ${COMPOSE_CMD[*]} logs -f" + echo " To stop: ${COMPOSE_CMD[*]} down" fi echo ""