Scaffolding: Federation sync app creation

.pre-commit-config.yaml

@@ -52,32 +52,44 @@ repos:
       hooks:
           - id: django-upgrade
             args: ["--target-version", "4.2"]
+            exclude: ^federation/
 
     # runs the ruff linter and formatter
     - repo: https://github.com/astral-sh/ruff-pre-commit
       rev: v0.15.15
       hooks:
           # linter
           - id: ruff-check # runs ruff check --force-exclude
+            exclude: ^federation/
             args:
-                # we have to pick one ruff config between gateway and sdk...
-                #   the SDK has additional constraints for having to support multiple
-                #   python versions, so it's a safer choice for a project-wide config;
-                #   though be aware that this might omit PyUpgrade improvements that
-                #   could be applied to the gateway side.
                 [
-                    # --verbose, # for debugging
-                    # --show-files, # for debugging
-                    # --show-fixes, # for debugging
                     --no-cache,
                     --fix,
                     --exit-non-zero-on-fix,
                     --config,
                     "sdk/pyproject.toml",
-                ] # --show-settings, # for debugging
+                ]
+          - id: ruff-check
+            alias: ruff-check-federation
+            name: ruff-check (federation)
+            files: ^federation/.*\.py$
+            args:
+                [
+                    --no-cache,
+                    --fix,
+                    --exit-non-zero-on-fix,
+                    --config,
+                    "federation/pyproject.toml",
+                ]
           # formatter
           - id: ruff-format # runs ruff format --force-exclude
+            exclude: ^federation/
             args: [--config, "sdk/pyproject.toml"]
+          - id: ruff-format
+            alias: ruff-format-federation
+            name: ruff-format (federation)
+            files: ^federation/.*\.py$
+            args: [--config, "federation/pyproject.toml"]
 
     # linter for django templates
     - repo: https://github.com/Riverside-Healthcare/djLint
@@ -100,7 +112,7 @@ repos:
       hooks:
           - id: deptry
             name: deptry
-            entry: bash -c "cd sdk && uv run deptry .; cd ../gateway && uv run deptry ."
+            entry: bash -c "cd sdk && uv run deptry .; cd ../gateway && uv run deptry .; cd ../federation && uv run deptry ."
             language: system
             pass_filenames: false # run once per commit, not per file
             types: [python]
@@ -110,6 +122,12 @@ repos:
             language: system
             pass_filenames: false
             files: ^gateway/sds_gateway/static/js/.*\.js$
+          - id: pyrefly-federation
+            name: pyrefly (federation)
+            entry: bash -c "cd federation && uv run pyrefly check -c pyproject.toml"
+            language: system
+            pass_filenames: false
+            files: ^federation/.*\.py$
 
 # sets up .pre-commit-ci.yaml to ensure pre-commit dependencies stay up to date
 ci:

federation/.envs/example/sync.env

@@ -0,0 +1,3 @@
+# Optional full channel override (default: federation:events:{site.name} from federation.toml)
+# FEDERATION_EVENTS_CHANNEL=
+FEDERATION_GATEWAY_API_KEY=

federation/.gitignore

@@ -0,0 +1,8 @@
+# Dev PKI material — generate with scripts/generate-dev-certs.sh
+
+__pycache__/
+
+federation.toml
+
+.envs/*
+!/.envs/example

federation/Dockerfile

@@ -0,0 +1,15 @@
+FROM docker.io/python:3.13-slim-trixie
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+WORKDIR /app
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+COPY pyproject.toml ./
+RUN uv sync --no-dev
+
+COPY sds_federation ./sds_federation
+
+EXPOSE 8000
+CMD ["uv", "run", "uvicorn", "sds_federation.main:app", "--host", "0.0.0.0", "--port", "8000"]

federation/certs/.gitignore

@@ -0,0 +1,3 @@
+# Dev PKI material — generate with scripts/generate-dev-certs.sh
+*
+!.gitignore

federation/compose.local.yaml

@@ -0,0 +1,44 @@
+networks:
+  sds-network-local:
+    external: true
+    name: sds-network-local
+  sds-network-local-opensearch:
+    driver: bridge
+    name: sds-network-local-opensearch
+
+services:
+  sds-federation-local-sync:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: sds-federation-local-sync
+    container_name: sds-federation-local-sync
+    env_file:
+      - .envs/local/sync.env
+      - ../federation-shared.env
+    environment:
+      FEDERATION_CONFIG_PATH: /etc/sds/federation.toml
+      GATEWAY_INTERNAL_BASE_URL: http://sds-gateway-local-app:8000/api/v1
+      REDIS_URL: redis://sds-gateway-local-redis:6379/0
+      OPENSEARCH_HOST: sds-gateway-local-opensearch
+    volumes:
+      - ./federation.toml:/etc/sds/federation.toml:ro
+      - ./certs:/etc/sds/certs:ro
+    ports:
+      - "8001:8000"
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "uv",
+          "run",
+          "python",
+          "-c",
+          "import sys, httpx; r=httpx.get('http://127.0.0.1:8000/health', timeout=4); sys.exit(0 if r.status_code==200 else 1)",
+        ]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+    networks:
+      - sds-network-local
+      - sds-network-local-opensearch

federation/compose.production.yaml

@@ -0,0 +1,44 @@
+networks:
+  sds-network-prod:
+    external: true
+    name: sds-network-prod
+  sds-network-prod-opensearch:
+    driver: bridge
+    name: sds-network-prod-opensearch
+
+services:
+  sds-federation-prod-sync:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: sds-federation-prod-sync
+    container_name: sds-federation-prod-sync
+    env_file:
+      - .envs/production/sync.env
+      - ../federation-shared.env
+    environment:
+      FEDERATION_CONFIG_PATH: /etc/sds/federation.toml
+      GATEWAY_INTERNAL_BASE_URL: http://sds-gateway-prod-app:8000/api/v1
+      REDIS_URL: redis://sds-gateway-prod-redis:6379/0
+      OPENSEARCH_HOST: sds-gateway-prod-opensearch
+    volumes:
+      - ./federation.toml:/etc/sds/federation.toml:ro
+      - ./certs:/etc/sds/certs:ro
+    ports:
+      - "8001:8000"
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "uv",
+          "run",
+          "python",
+          "-c",
+          "import sys, httpx; r=httpx.get('http://127.0.0.1:8000/health', timeout=4); sys.exit(0 if r.status_code==200 else 1)",
+        ]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+    networks:
+      - sds-network-prod
+      - sds-network-prod-opensearch

federation/federation.example.toml

@@ -0,0 +1,14 @@
+[site]
+name = "your-site"
+fqdn = "your-site.domain"
+display_name = "Your Site"
+
+[[peers]]
+# Notre Dame SDS site is the original site deployment
+# and is included as a peer by default.
+name = "crc"
+fqdn = "sds.crc.nd.edu"
+display_name = "Notre Dame CRC"
+gateway_api_base = "https://sds.crc.nd.edu/api/v1"
+sync_service_url = "https://sds.crc.nd.edu/sync/"
+ca_cert_path = "/etc/sds/certs/crc-ca.pem"

federation/justfile

@@ -0,0 +1,101 @@
+set shell := ["bash", "-eu", "-o", "pipefail", "-c"]
+
+# TIP: Production on this machine? Add hostname to ../gateway/scripts/prod-hostnames.env
+#       or set SDS_ENV=production|local
+
+env_selection_script := "./scripts/env-selection.sh"
+
+# variables | run `just env` to see current values
+
+compose_file := shell(env_selection_script + ' $1', "compose_file")
+env := shell(env_selection_script + ' $1', "env")
+env_file := shell(env_selection_script + ' $1', "env_file")
+sync_container := shell(env_selection_script + ' $1', "sync_container")
+docker_compose := "COMPOSE_FILE=" + compose_file + " docker compose --env-file " + env_file
+
+alias run := up
+
+default:
+    @just --list
+
+# prints currently selected environment (override with SDS_ENV=local|production)
+[group('utilities')]
+env:
+    #!/usr/bin/env bash
+    echo -e "\nSelected env:\n"
+    echo -e "\tEnvironment: \e[34m             '{{ env }}'\e[0m"
+    echo -e "\tEnvironment file: \e[34m        '{{ env_file }}'\e[0m"
+    echo -e "\tCompose file: \e[34m            '{{ compose_file }}'\e[0m"
+    echo -e "\tSync container: \e[34m          '{{ sync_container }}'\e[0m"
+    echo -e "\tDocker compose command: \e[34m  '{{ docker_compose }}'\e[0m"
+
+    if ! [ -f "{{ compose_file }}" ]; then
+        echo -e "\n\e[31mError:\e[0m Compose file '{{ compose_file }}' does not exist."
+        exit 1
+    fi
+    if ! [ -f "{{ env_file }}" ]; then
+        echo -e "\n\e[33mWarning:\e[0m Env file '{{ env_file }}' does not exist."
+        echo -e "Copy from .envs/example/sync.env if needed."
+    fi
+
+# install runtime + dev deps (pytest)
+[group('setup')]
+sync:
+    uv sync --extra dev
+
+[group('qa')]
+test-regression +args='':
+    uv run pytest -m regression {{ args }}
+
+[group('qa')]
+test-integration +args='':
+    uv run pytest -m integration {{ args }}
+
+[group('qa')]
+test +args='':
+    uv run pytest {{ args }}
+
+[group('qa')]
+test-q:
+    uv run pytest -q
+
+# Run federation pre-commit hooks from repo root (requires gateway dev deps / pre-commit install)
+[group('qa')]
+pre-commit +args='':
+    cd .. && uv run --directory gateway --extra local pre-commit run ruff-check-federation --all-files {{ args }}
+    cd .. && uv run --directory gateway --extra local pre-commit run ruff-format-federation --all-files {{ args }}
+    cd .. && uv run --directory gateway --extra local pre-commit run pyrefly-federation --all-files {{ args }}
+
+# dev PKI for mTLS experiments (see docs/mtls-certificates.md)
+[group('setup')]
+gen-certs +args='':
+    ./scripts/generate-dev-certs.sh all {{ args }}
+
+[group('setup')]
+gen-certs-ca:
+    ./scripts/generate-dev-certs.sh ca
+
+# publish simulated federation:events to Redis
+[group('dev')]
+simulate-redis +args='':
+    REDIS_URL="${REDIS_URL:-redis://localhost:6379/0}" uv run python scripts/simulate_redis_event.py {{ args }}
+
+[group('docker')]
+build +args='':
+    {{ docker_compose }} build {{ args }}
+
+[group('docker')]
+up +args='':
+    {{ docker_compose }} up -d --remove-orphans {{ args }}
+
+[group('docker')]
+down +args='':
+    {{ docker_compose }} down {{ args }}
+
+[group('docker')]
+logs +args='':
+    {{ docker_compose }} logs --tail 1000 -f {{ args }}
+
+[group('docker')]
+dc +args='':
+    {{ docker_compose }} {{ args }}