testing branch for new girder_worker_api_url environment variable

.env.default

@@ -25,7 +25,11 @@ COMPOSE_PROFILES=gpu
 #RABBITMQ_DEFAULT_VHOST=default
 #GIRDER_WORKER_BROKER=amqp://guest:guest@rabbit/default
 #GIRDER_WORKER_BACKEND=rpc://guest:guest@localhost/
+# Server-side: stamped into jobs at schedule time (worker.api_url setting)
 #GIRDER_SETTING_WORKER_API_URL=http://girder:8080/api/v1
+# Worker-side: per-worker override when this process cannot reach the stamped URL
+# (e.g. remote workers: http://WEB_HOST:8010/api/v1). Leave unset on single-node.
+#GIRDER_WORKER_API_URL=
 #GIRDER_NOTIFICATION_REDIS_URL=redis://redis:6379
 
 # Docker image tag for kitware/viame-web (workers use cpu/gpu tags in compose)
@@ -66,7 +70,7 @@ COMPOSE_PROFILES=gpu
 #ACME_EMAIL=changeme@domain.com
 #ACME_CA_SERVER=https://acme-v02.api.letsencrypt.org/directory
 #LOG_LEVEL=ERROR
-# Split or standalone worker: remote Girder API (replaces legacy WORKER_API_URL)
-#GIRDER_SETTING_WORKER_API_URL=https://viame.kitware.com/api/v1
+# Split / remote worker: override stamped API URL on the worker process
+#GIRDER_WORKER_API_URL=https://viame.kitware.com/api/v1
 #SOCK_PATH=/var/run/docker.sock
 WATCHTOWER_API_TOKEN="customtokenstring"

docker-compose.yml

@@ -77,6 +77,7 @@ services:
       - "GIRDER_MONGO_URI=mongodb://mongo:27017/girder"
       - "GIRDER_NOTIFICATION_REDIS_URL=redis://redis:6379"
       - "C_FORCE_ROOT=1"
+      - "GIRDER_WORKER_API_URL=${GIRDER_WORKER_API_URL:-}"
 
 # Memcache is used for caching girder-large-images, should auto configure with it being present
   memcached:
@@ -144,6 +145,7 @@ services:
       - "GIRDER_WORKER_BROKER=${GIRDER_WORKER_BROKER:-amqp://guest:guest@rabbit/default}"
       - "GIRDER_WORKER_BACKEND=${GIRDER_WORKER_BACKEND:-rpc://guest:guest@localhost/}"
       - "GIRDER_SETTING_WORKER_API_URL=${GIRDER_SETTING_WORKER_API_URL:-http://girder:8080/api/v1}"
+      - "GIRDER_WORKER_API_URL=${GIRDER_WORKER_API_URL:-}"
 
   girder_worker_pipelines:
     # Merge base-worker object with this config
@@ -173,6 +175,7 @@ services:
       - "CELERY_BROKER_URL=${CELERY_BROKER_URL:-amqp://guest:guest@rabbit/default}"
       - "GIRDER_WORKER_BACKEND=${GIRDER_WORKER_BACKEND:-rpc://guest:guest@localhost/}"
       - "GIRDER_SETTING_WORKER_API_URL=${GIRDER_SETTING_WORKER_API_URL:-http://girder:8080/api/v1}"
+      - "GIRDER_WORKER_API_URL=${GIRDER_WORKER_API_URL:-}"
       - "GIRDER_MONGO_URI=mongodb://mongo:27017/girder"
       - "GIRDER_NOTIFICATION_REDIS_URL=redis://redis:6379"
     healthcheck:
@@ -209,6 +212,7 @@ services:
       - "CELERY_BROKER_URL=${CELERY_BROKER_URL:-amqp://guest:guest@rabbit/default}"
       - "GIRDER_WORKER_BACKEND=${GIRDER_WORKER_BACKEND:-rpc://guest:guest@localhost/}"
       - "GIRDER_SETTING_WORKER_API_URL=${GIRDER_SETTING_WORKER_API_URL:-http://girder:8080/api/v1}"
+      - "GIRDER_WORKER_API_URL=${GIRDER_WORKER_API_URL:-}"
     healthcheck:
       test: ["CMD", "nvidia-smi"]
       interval: 15m

docs/Deployment-Docker-Compose.md

@@ -119,32 +119,63 @@ It's possible to split your web server and task runner between multiple nodes.
 * Make two cloud VM instances, one with NVIDIA drivers and container toolkit, and one without.  This is still a special case of scenario 1 from the [Provisioning Guide](Deployment-Provision.md)
 * Clone the dive repository on both VMs. The `.env` files are **not** identical — the web VM keeps the internal Compose service names, while the worker VM points at the web VM's IP or hostname (referred to below as `WEB_HOST`).
 
+### Worker API URL settings
+
+Two related variables control how workers call back to Girder. They apply at different layers:
+
+| Variable | Where to set it | Role |
+|----------|-----------------|------|
+| `GIRDER_SETTING_WORKER_API_URL` | **Girder web server** process | Sets the Girder system setting `worker.api_url`. That value is **stamped into every job** when it is scheduled (Celery headers / `jobInfoSpec`). |
+| `GIRDER_WORKER_API_URL` | **Worker** process only | Optional **per-worker override**. When set, the worker uses this URL for API callbacks instead of the URL stamped at schedule time (including job status/log updates). |
+
+!!! tip "When to set `GIRDER_WORKER_API_URL`"
+
+    Use it when a worker cannot reach the API URL stamped by the server — typically remote or external workers that cannot resolve Docker-internal names like `girder:8080`.
+
+    Typical mixed setup:
+
+    * Web VM + `localworker` on the Compose network keep the default stamped URL `http://girder:8080/api/v1`.
+    * Remote worker VMs set `GIRDER_WORKER_API_URL=http://WEB_HOST:8010/api/v1` so they reach Girder through the host-published Traefik port.
+
+    Leave `GIRDER_WORKER_API_URL` **unset** on single-node stacks and on any worker that can already reach the stamped URL.
+
 !!! warning
 
-    `GIRDER_SETTING_WORKER_API_URL` is applied by the **web server** as the `worker.api_url` Girder system setting and is then stamped into every job that is dispatched to workers. It must be set on the **web VM**. Setting it only on the worker VM has no effect on the callback URL that workers actually use.
+    `GIRDER_SETTING_WORKER_API_URL` only affects the **web server** process that schedules jobs. Setting it on a worker container does **not** change the callback URL that worker uses at runtime — use `GIRDER_WORKER_API_URL` for that.
+
+    Port `8080` is Girder's in-container port and is **not** published on the host. Traefik publishes the API on host port `8010` (base `docker-compose.yml`) or on `80`/`443` when you also use `docker-compose.prod.yml`. Remote workers should use the published port — never `8080` unless you publish it yourself.
 
 ### Web VM `.env`
 
-The web VM runs RabbitMQ, Redis, and Mongo as Compose services, so it keeps the internal service names — with one exception: `GIRDER_SETTING_WORKER_API_URL` must be an address the **remote worker** can reach, not the internal `girder:8080`.
+The web VM runs RabbitMQ, Redis, and Mongo as Compose services and should keep internal service names. For a mixed local + remote worker deployment, leave the server stamping the internal URL (the Compose default):
 
 ```
-GIRDER_SETTING_WORKER_API_URL=http://WEB_HOST:8010/api/v1
+# Optional — this is already the Compose default for a single-node / mixed stack
+#GIRDER_SETTING_WORKER_API_URL=http://girder:8080/api/v1
 ```
 
-!!! note
+`localworker` on the same Compose network can reach `girder:8080` and does not need `GIRDER_WORKER_API_URL`.
 
-    Port `8080` is Girder's in-container port and is **not** published on the host. Traefik publishes the API on host port `8010` (base `docker-compose.yml`) or on `80`/`443` when you also use `docker-compose.prod.yml`. Use the published port here — never `8080`.
+!!! note
 
-    This only applies when workers run on a **separate** host. For a single-node (all-in-one) stack, leave `GIRDER_SETTING_WORKER_API_URL` unset — the default `http://girder:8080/api/v1` is correct because the worker resolves `girder` and reaches port `8080` over the internal Compose network.
+    If **every** worker (including `localworker`) can reach a single public API URL — for example via host hairpin to `WEB_HOST:8010` — you may instead set `GIRDER_SETTING_WORKER_API_URL=http://WEB_HOST:8010/api/v1` on the web VM and omit `GIRDER_WORKER_API_URL`. Prefer `GIRDER_WORKER_API_URL` on remote workers when local and remote workers need different reachable URLs.
 
 ### Worker VM `.env`
 
 On the worker VM, uncomment and set these to point at `WEB_HOST`:
 
 * `GIRDER_WORKER_BROKER` — RabbitMQ URL on the web VM (e.g. `amqp://guest:guest@WEB_HOST/default`)
-* `GIRDER_SETTING_WORKER_API_URL` — Girder API URL on the web VM (e.g. `http://WEB_HOST:8010/api/v1`; match the web VM's value)
+* `GIRDER_WORKER_API_URL` — Girder API URL reachable from this worker (e.g. `http://WEB_HOST:8010/api/v1`)
 * `GIRDER_NOTIFICATION_REDIS_URL` — Redis URL on the web VM (e.g. `redis://WEB_HOST:6379`); required so workers can publish job/UI status notifications
 
+Example:
+
+```
+GIRDER_WORKER_BROKER=amqp://guest:guest@WEB_HOST/default
+GIRDER_WORKER_API_URL=http://WEB_HOST:8010/api/v1
+GIRDER_NOTIFICATION_REDIS_URL=redis://WEB_HOST:6379
+```
+
 ### Required connectivity
 
 The worker VM must be able to reach the web VM on these ports (port `8080` does **not** need to be open between VMs):
@@ -169,13 +200,24 @@ In this split setup, `localworker` on the web server handles the `local` queue,
 
 ### Verify the configuration
 
-After the stack is up, confirm the web server is handing out the correct callback URL (not the internal `girder:8080` default). From the web VM's Swagger page or via curl:
+After the stack is up:
+
+1. Confirm what the web server stamps into jobs (Swagger or curl on the web VM):
 
 ```
 GET /api/v1/system/setting?key=worker.api_url
 ```
 
-This must return your `http://WEB_HOST:8010/api/v1` value. If it still shows `http://girder:8080/api/v1`, the web VM's `GIRDER_SETTING_WORKER_API_URL` was not applied — jobs will appear to dispatch (and `GET /worker/status` will look healthy, since that only reflects broker connectivity) but will stall because remote workers cannot reach `girder:8080`.
+For the recommended mixed setup this is `http://girder:8080/api/v1`. That value is correct for `localworker`; remote workers rely on `GIRDER_WORKER_API_URL` instead.
+
+2. On each remote worker container, confirm the override is present:
+
+```bash
+docker compose exec girder_worker_default printenv GIRDER_WORKER_API_URL
+# -> http://WEB_HOST:8010/api/v1
+```
+
+If remote workers stall while `GET /worker/status` looks healthy (broker only), they usually cannot reach the stamped URL and `GIRDER_WORKER_API_URL` is missing or wrong.
 
 ## Addon management
 
@@ -233,7 +275,7 @@ This image contains both the backend and client.
 | GIRDER_ADMIN_PASS | `letmein` | admin password |
 | GIRDER_WORKER_BROKER | `amqp://guest:guest@rabbit/default` | RabbitMQ connection string (Celery broker) |
 | GIRDER_WORKER_BACKEND | `rpc://guest:guest@localhost/` | Celery result backend (RPC) |
-| GIRDER_SETTING_WORKER_API_URL | `http://girder:8080/api/v1` | Girder REST API URL used by workers. Default is correct for a single-node stack; override with the web host's published URL (e.g. `http://WEB_HOST:8010/api/v1`) only for [split deployments](#splitting-services). |
+| GIRDER_SETTING_WORKER_API_URL | `http://girder:8080/api/v1` | Girder system setting `worker.api_url` stamped into jobs at schedule time. Default is correct for single-node and for `localworker` on the Compose network. See [Worker API URL settings](#worker-api-url-settings). |
 | GIRDER_NOTIFICATION_REDIS_URL | `redis://redis:6379` | Redis URL for notification fan-out |
 | GIRDER_STATIC_ROOT_DIR | `/opt/dive/clients/girder` | Built web client static files (set in image/Compose) |
 
@@ -260,7 +302,8 @@ This image contains a celery worker to run VIAME pipelines and transcoding jobs.
 | WORKER_CONCURRENCY | `# of CPU cores` | max concurrnet jobs. **Lower this if you run training** |
 | WORKER_GPU_UUID | null | leave empty to use all GPUs.  Specify UUID to use specific device |
 | GIRDER_WORKER_BROKER | `amqp://guest:guest@rabbit/default` | RabbitMQ connection string. Ignored in standalone mode. |
-| GIRDER_SETTING_WORKER_API_URL | `http://girder:8080/api/v1` | Girder API URL (split/multi-node deployments). Ignored in standalone mode when using `DIVE_API_URL`. |
+| GIRDER_WORKER_API_URL | unset | Optional per-worker override of the Girder API base URL used for callbacks. Set on remote/external workers that cannot reach the URL stamped by the server (e.g. `http://WEB_HOST:8010/api/v1`). See [Worker API URL settings](#worker-api-url-settings). Ignored in standalone mode when using `DIVE_API_URL`. |
+| GIRDER_SETTING_WORKER_API_URL | `http://girder:8080/api/v1` | Only meaningful on the **web server** (stamped at schedule time). Setting it on a worker process does not override callback URLs at runtime — use `GIRDER_WORKER_API_URL` instead. |
 | GIRDER_NOTIFICATION_REDIS_URL | `redis://redis:6379` | Redis for notifications when running workers in Compose |
 | KWIVER_DEFAULT_LOG_LEVEL | `warn` | Log level for VIAME pipeline jobs (env name unchanged; used by the Kwiver logging stack) |
 | DIVE_USERNAME | null | Username to start private queue processor. Providing this enables standalone mode. |

docs/Deployment-Girder-5-Upgrade.md

@@ -21,12 +21,15 @@ Update your `.env` (and any external orchestration) to use Girder 5 names:
 | Girder 4 / legacy name | Girder 5 name | Purpose |
 |------------------------|---------------|---------|
 | `CELERY_BROKER_URL` | `GIRDER_WORKER_BROKER` | RabbitMQ URL for Celery (message broker). |
-| `WORKER_API_URL` | `GIRDER_SETTING_WORKER_API_URL` | Girder REST API URL workers call (e.g. `http://girder:8080/api/v1`). |
+| `WORKER_API_URL` | `GIRDER_SETTING_WORKER_API_URL` | Server-side Girder setting `worker.api_url` stamped into jobs at schedule time (e.g. `http://girder:8080/api/v1`). |
+| *(not used)* | `GIRDER_WORKER_API_URL` | Optional **worker-side** override of the stamped API URL for callbacks. Set on remote workers that cannot reach the server-stamped URL. |
 | *(not used)* | `GIRDER_WORKER_BACKEND` | Celery result backend (Compose default: `rpc://guest:guest@localhost/`). |
 | *(not used)* | `GIRDER_NOTIFICATION_REDIS_URL` | Redis URL for Girder notification fan-out (e.g. `redis://redis:6379`). |
 | *(not used)* | `GIRDER_STATIC_ROOT_DIR` | Path to built web client inside the Girder container (set in Compose). |
 
-`GIRDER_WORKER_BACKEND` is **not** a replacement for `WORKER_API_URL`. Do not point it at `http://…/api/v1`; use `GIRDER_SETTING_WORKER_API_URL` for that.
+`GIRDER_WORKER_BACKEND` is **not** a replacement for `WORKER_API_URL`. Do not point it at `http://…/api/v1`.
+
+Use `GIRDER_SETTING_WORKER_API_URL` on the **web server** (what gets stamped into jobs). Use `GIRDER_WORKER_API_URL` on **worker** processes when they need a different reachable API base than the stamped value. Details: [Worker API URL settings](Deployment-Docker-Compose.md#worker-api-url-settings).
 
 ## Recommended upgrade steps
 
@@ -35,7 +38,7 @@ Update your `.env` (and any external orchestration) to use Girder 5 names:
 3. **Set image tag** if needed (for example `TAG=latest` in `.env`), then pull images.
 4. **Add Redis** — use the `redis` service from `docker-compose.yml` or an external Redis instance and set `GIRDER_NOTIFICATION_REDIS_URL` on `girder` and all worker services.
 5. **Rename variables** in `.env` per the table above.
-6. **Multi-node deployments:** set `GIRDER_WORKER_BROKER` to your RabbitMQ URL and `GIRDER_SETTING_WORKER_API_URL` to your Girder API URL on every worker host; workers must reach Redis used by the web tier if they rely on notification-related features configured in Compose.
+6. **Multi-node deployments:** set `GIRDER_WORKER_BROKER` to your RabbitMQ URL and `GIRDER_WORKER_API_URL` to a Girder API URL reachable from that worker (for example `http://WEB_HOST:8010/api/v1`). Keep `GIRDER_SETTING_WORKER_API_URL` on the web VM at the value appropriate for `localworker` (often the Compose default `http://girder:8080/api/v1`). Workers must reach Redis used by the web tier if they rely on notification-related features configured in Compose.
 7. **Rebuild or pull** images, then restart the stack:
 
 ```bash

test-girder-worker-api-url/Dockerfile.girder

@@ -0,0 +1,75 @@
+# Test build of kitware/viame-web with Girder from girder-worker-api-override.
+# Based on docker/girder.Dockerfile; installs Girder packages from the test branch.
+
+# ========================
+# == CLIENT BUILD STAGE ==
+# ========================
+FROM node:24.15.0 AS client-builder
+WORKDIR /app
+
+COPY client/package.json client/package-lock.json /app/
+RUN npm install
+COPY .git/ /app/.git/
+COPY client/ /app/
+RUN npm run build:web
+
+FROM node:24.15.0 AS girder-client-builder
+WORKDIR /app
+RUN apt-get update && apt-get install -y git
+ARG GIRDER_GIT_REPO=https://github.com/girder/girder.git
+ARG GIRDER_GIT_REF=girder-worker-api-override
+ARG CACHEBUST=1
+RUN git clone --depth 1 --branch "${GIRDER_GIT_REF}" "${GIRDER_GIT_REPO}" girder-src \
+  && mv girder-src girder
+RUN cd girder/girder/web && npm install --include=optional && npx vite build --base=/girder/
+
+# ========================
+# == SERVER BUILD STAGE ==
+# ========================
+FROM ghcr.io/astral-sh/uv:python3.11-bookworm-slim AS server-builder
+SHELL ["/bin/bash", "-c"]
+
+WORKDIR /opt/dive/src
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+      curl git build-essential libssl-dev libffi-dev python3-libtiff \
+      libgdal-dev python3-dev cargo nodejs npm && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy
+ENV VIRTUAL_ENV="/opt/dive/local/venv"
+ENV UV_PROJECT_ENVIRONMENT=/opt/dive/local/venv
+ENV PATH="/opt/dive/local/venv/bin:$PATH"
+RUN uv venv /opt/dive/local/venv
+
+COPY server/pyproject.toml server/uv.lock /opt/dive/src/
+RUN uv sync --frozen --no-install-project --no-dev --extra large-image
+COPY server/ /opt/dive/src/
+RUN uv sync --frozen --no-dev --extra large-image
+
+# Override locked girder-worker with the test branch (adds GIRDER_WORKER_API_URL).
+ARG GIRDER_GIT_REPO=https://github.com/girder/girder.git
+ARG GIRDER_GIT_REF=girder-worker-api-override
+COPY test-girder-worker-api-url/install_girder_branch.sh /tmp/install_girder_branch.sh
+RUN chmod +x /tmp/install_girder_branch.sh \
+  && GIRDER_GIT_REPO="${GIRDER_GIT_REPO}" GIRDER_GIT_REF="${GIRDER_GIT_REF}" \
+     /tmp/install_girder_branch.sh
+
+# =================
+# == DIST SERVER ==
+# =================
+FROM ghcr.io/astral-sh/uv:python3.11-bookworm-slim AS server
+
+ENV GIT_PYTHON_REFRESH="quiet"
+ENV VIRTUAL_ENV="/opt/dive/local/venv"
+ENV UV_PROJECT_ENVIRONMENT=/opt/dive/local/venv
+ENV PATH="/opt/dive/local/venv/bin:/usr/local/bin:$PATH"
+
+COPY --from=server-builder /opt/dive/local/venv /opt/dive/local/venv
+COPY --from=server-builder /opt/dive/src /opt/dive/src
+COPY --from=client-builder /app/dist/ /opt/dive/clients/dive
+COPY --from=girder-client-builder /app/girder/girder/web/dist/ /opt/dive/clients/girder
+COPY docker/entrypoint_server.sh docker/server_setup.py /
+RUN export GIRDER_STATIC_ROOT_DIR=/opt/dive/clients/girder
+ENTRYPOINT [ "/entrypoint_server.sh" ]

test-girder-worker-api-url/Dockerfile.worker

@@ -0,0 +1,50 @@
+# Test build of kitware/viame-worker:cpu with Girder from girder-worker-api-override.
+# Based on docker/girder_worker.Dockerfile; installs Girder packages from the test branch.
+
+FROM python:3.11-bookworm AS worker
+
+RUN apt-get update && \
+  apt-get install -qy tini ffmpeg git && \
+  apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN ln -sf /usr/bin/tini /tini
+RUN ln -fs /usr/bin/python3.11 /usr/bin/python
+WORKDIR /opt/dive/src
+
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+ENV VIRTUAL_ENV="/opt/dive/local/venv"
+ENV UV_PROJECT_ENVIRONMENT=/opt/dive/local/venv
+ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy
+RUN uv venv /opt/dive/local/venv
+ENV PATH="/opt/dive/local/venv/bin:/usr/local/bin:$PATH"
+
+COPY server/pyproject.toml server/uv.lock /opt/dive/src/
+RUN uv sync --frozen --no-install-project --no-dev
+
+COPY server/ /opt/dive/src/
+
+RUN useradd --create-home --uid 1099 --shell=/bin/bash dive
+RUN install -g dive -o dive -d /tmp/addons
+RUN chown -R dive /opt/dive/local/
+
+USER dive
+RUN uv sync --frozen --no-dev
+
+# Override locked girder-worker with the test branch (adds GIRDER_WORKER_API_URL).
+ARG GIRDER_GIT_REPO=https://github.com/girder/girder.git
+ARG GIRDER_GIT_REF=girder-worker-api-override
+USER root
+COPY test-girder-worker-api-url/install_girder_branch.sh /tmp/install_girder_branch.sh
+RUN chmod +x /tmp/install_girder_branch.sh \
+  && chown dive:dive /tmp/install_girder_branch.sh
+USER dive
+RUN GIRDER_GIT_REPO="${GIRDER_GIT_REPO}" GIRDER_GIT_REF="${GIRDER_GIT_REF}" \
+    /tmp/install_girder_branch.sh
+
+RUN install -d /opt/dive/local/ffmpeg && \
+  ln -sf /usr/bin/ffmpeg /opt/dive/local/ffmpeg/ffmpeg && \
+  ln -sf /usr/bin/ffprobe /opt/dive/local/ffmpeg/ffprobe
+COPY --chown=dive:dive docker/entrypoint_worker.sh /
+
+ENTRYPOINT ["/tini", "--"]
+CMD ["/entrypoint_worker.sh"]