#!/usr/bin/env bash set -euo pipefail DOCKER_SMOKE_RUN_ID="${DOCKER_SMOKE_RUN_ID:-$$}" if [[ -z "$DOCKER_SMOKE_RUN_ID" || ${#DOCKER_SMOKE_RUN_ID} -gt 80 || ! $DOCKER_SMOKE_RUN_ID =~ ^[A-Za-z0-9_.-]+$ ]]; then echo "DOCKER_SMOKE_RUN_ID must contain 1-80 Docker-safe characters" >&2 exit 2 fi # shellcheck source=scripts/lib/docker-rls-role-fixture.sh source "$(dirname "${BASH_SOURCE[0]}")/lib/docker-rls-role-fixture.sh" # shellcheck source=scripts/lib/docker-runtime-smoke.sh source "$(dirname "${BASH_SOURCE[0]}")/lib/docker-runtime-smoke.sh" IMAGE="${1:-hivegrid-worker:smoke}" MIGRATOR_IMAGE="${DOCKER_SMOKE_MIGRATOR_IMAGE:-hivegrid-web:smoke}" CONTAINER_NAME="${DOCKER_WORKER_SMOKE_CONTAINER:-hivegrid-worker-runtime-smoke-${DOCKER_SMOKE_RUN_ID}}" DB_CONTAINER_NAME="${DOCKER_WORKER_SMOKE_DB_CONTAINER:-hivegrid-worker-runtime-smoke-db-${DOCKER_SMOKE_RUN_ID}}" REDIS_CONTAINER_NAME="${DOCKER_WORKER_SMOKE_REDIS_CONTAINER:-hivegrid-worker-runtime-smoke-redis-${DOCKER_SMOKE_RUN_ID}}" MEILI_CONTAINER_NAME="${DOCKER_WORKER_SMOKE_MEILI_CONTAINER:-hivegrid-worker-runtime-smoke-meili-${DOCKER_SMOKE_RUN_ID}}" MEILI_IMAGE="${DOCKER_WORKER_SMOKE_MEILI_IMAGE:-getmeili/meilisearch:v1.8}" MEILI_IMAGE_PULL_TIMEOUT_SECONDS="${DOCKER_WORKER_SMOKE_MEILI_IMAGE_PULL_TIMEOUT_SECONDS:-180}" NETWORK_NAME="${DOCKER_WORKER_SMOKE_NETWORK:-hivegrid-worker-runtime-smoke-${DOCKER_SMOKE_RUN_ID}}" MIGRATION_CONTAINER_NAME="${DOCKER_WORKER_SMOKE_MIGRATION_CONTAINER:-hivegrid-worker-runtime-smoke-migrate-${DOCKER_SMOKE_RUN_ID}}" ASSERT_CONTAINER_NAME="${DOCKER_WORKER_SMOKE_ASSERT_CONTAINER:-hivegrid-worker-runtime-smoke-assert-${DOCKER_SMOKE_RUN_ID}}" DISABLED_HEALTH_CONTAINER_NAME="${DOCKER_WORKER_SMOKE_DISABLED_HEALTH_CONTAINER:-hivegrid-worker-runtime-smoke-disabled-${DOCKER_SMOKE_RUN_ID}}" ENABLED_HEALTH_CONTAINER_NAME="${DOCKER_WORKER_SMOKE_ENABLED_HEALTH_CONTAINER:-hivegrid-worker-runtime-smoke-enabled-${DOCKER_SMOKE_RUN_ID}}" MAIL_HEALTH_CONTAINER_NAME="${DOCKER_WORKER_SMOKE_MAIL_HEALTH_CONTAINER:-hivegrid-worker-runtime-smoke-mail-${DOCKER_SMOKE_RUN_ID}}" SMOKE_REVISION="${DOCKER_WORKER_SMOKE_REVISION:-}" DB_PASSWORD="hivegrid" DB_NAME="hivegrid" DB_USER="hivegrid" WORKER_DB_PASSWORD="hivegrid-worker-smoke" APP_DB_PASSWORD="hivegrid-app-smoke" MIGRATOR_DB_PASSWORD="hivegrid-migrator-smoke" AUTHENTIK_JIT_DB_PASSWORD="hivegrid-authentik-jit-smoke" cleanup() { docker_smoke_cleanup_container "worker:cleanup-containers" \ "$CONTAINER_NAME" "$MIGRATION_CONTAINER_NAME" "$ASSERT_CONTAINER_NAME" \ "$DISABLED_HEALTH_CONTAINER_NAME" "$ENABLED_HEALTH_CONTAINER_NAME" "$MAIL_HEALTH_CONTAINER_NAME" \ "$DB_CONTAINER_NAME" "$REDIS_CONTAINER_NAME" "$MEILI_CONTAINER_NAME" docker_smoke_cleanup_network "worker:cleanup-network" "$NETWORK_NAME" } resolve_smoke_revision() { local image_revision if [[ -n "$SMOKE_REVISION" ]]; then printf "%s" "$SMOKE_REVISION" return fi docker_smoke_capture image_revision "worker:inspect-revision" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \ docker image inspect -f '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$IMAGE" printf "%s" "$image_revision" } verify_migrator_revision() { local migrator_revision docker_smoke_capture migrator_revision "worker:inspect-migrator-revision" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \ docker image inspect -f '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$MIGRATOR_IMAGE" if [[ "$migrator_revision" != "$SMOKE_REVISION" ]]; then echo "Migrator image ${MIGRATOR_IMAGE} revision ${migrator_revision:-} does not match worker image revision ${SMOKE_REVISION}" >&2 exit 1 fi } docker_smoke_install_cleanup_traps cleanup cleanup SMOKE_REVISION="$(resolve_smoke_revision)" if [[ -z "$SMOKE_REVISION" || "$SMOKE_REVISION" == "" || "$SMOKE_REVISION" == "unknown" ]]; then echo "Image ${IMAGE} does not expose a concrete org.opencontainers.image.revision; build with --build-arg APP_REVISION or set DOCKER_WORKER_SMOKE_REVISION explicitly" >&2 exit 1 fi verify_migrator_revision worker_env=( -e NODE_ENV=production -e APP_NAME=HiveGrid -e APP_BASE_DOMAIN=localhost -e DATABASE_RUNTIME_ROLE=hivegrid_worker -e RLS_RUNTIME_MODE=required -e AUDIT_CHAIN_SECRET=deploy-smoke-audit-chain-secret-minimum-32 -e ADMIN_CONNECTION_MASTER_KEY=deploy-smoke-admin-connection-key-minimum-32 ) # Obvious non-production fixtures prove that the required general-worker # provider boundary is wired without storing any live credential in the repo. product_acquisition_env=( -e LAGO_API_URL=http://lago.internal.test:3108 -e LAGO_API_KEY=test-lago-runtime-smoke -e LAGO_TIMEOUT_MS=10000 -e MOLLIE_API_KEY=test-mollie-runtime-smoke -e MOLLIE_PROFILE_ID=pfl_runtime_smoke -e BILLING_MOLLIE_WEBHOOK_URL=https://api.hivegrid.test/v1/billing/mollie/webhooks -e MOLLIE_TIMEOUT_MS=10000 ) # Obvious non-production fixtures keep the general-worker smoke aligned with # its fail-closed export-retention and tenant-hard-delete contracts. The # delete-only runtime must not receive the SSE-C key used by writers/readers. general_hard_delete_env=( -e EXPORT_RETENTION_QUEUE_ENABLED=true -e STORAGE_ENDPOINT=https://storage.example.test -e STORAGE_REGION=eu-central-1 -e STORAGE_BUCKET=docker-smoke-private-exports -e STORAGE_ACCESS_KEY_ID=docker-smoke-retention-access-key -e STORAGE_SECRET_ACCESS_KEY=docker-smoke-retention-secret-key -e STORAGE_FORCE_PATH_STYLE=false -e MEILI_URL=http://meili:7700 -e MEILI_API_KEY=docker-smoke-tenant-delete-meili-master-key ) expect_worker_health() { local output="$1" local expected_status="$2" local expected_enabled="$3" local expected_group="${4:-}" local expected_queue="${5:-}" node -e ' const [raw, expectedStatus, expectedEnabled, expectedRevision, expectedGroup, expectedQueue] = process.argv.slice(1); const report = JSON.parse(raw); if (report.status !== expectedStatus) { console.error(`Expected worker status ${expectedStatus}, got ${report.status}`); process.exit(1); } if (String(report.workerEnabled) !== expectedEnabled) { console.error(`Expected workerEnabled ${expectedEnabled}, got ${report.workerEnabled}`); process.exit(1); } if (report.revision !== expectedRevision) { console.error(`Expected revision ${expectedRevision}, got ${report.revision || ""}`); process.exit(1); } if (expectedGroup && report.workerGroup !== expectedGroup) { console.error(`Expected workerGroup ${expectedGroup}, got ${report.workerGroup || ""}`); process.exit(1); } if (expectedQueue && !Array.isArray(report.queues)) { console.error("Expected worker health queues to be an array"); process.exit(1); } if (expectedQueue && !report.queues.includes(expectedQueue)) { console.error(`Expected worker queues to include ${expectedQueue}, got ${report.queues?.join(",") || ""}`); process.exit(1); } if (expectedStatus === "ready" && (report.databaseStatus !== "ready" || report.redisStatus !== "ready")) { console.error(`Expected ready DB/Redis, got database=${report.databaseStatus} redis=${report.redisStatus}`); process.exit(1); } ' "$output" "$expected_status" "$expected_enabled" "$SMOKE_REVISION" "$expected_group" "$expected_queue" } expect_worker_deploy_health() { local output="$1" local expected_group="$2" node -e ' const [raw, expectedRevision, expectedGroup] = process.argv.slice(1); const report = JSON.parse(raw); if (report.status !== "ready" || report.revision !== expectedRevision || report.workerGroup !== expectedGroup) { console.error("Worker deployment health did not prove the expected revision and group"); process.exit(1); } ' "$output" "$SMOKE_REVISION" "$expected_group" } docker_smoke_run "worker:create-network" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" \ docker network create "$NETWORK_NAME" >/dev/null health_output="" docker_smoke_capture health_output "worker:health-disabled" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" \ docker run --rm --name "$DISABLED_HEALTH_CONTAINER_NAME" \ "${worker_env[@]}" \ -e WORKER_ENABLED=false \ "$IMAGE" node apps/worker/dist/health.js expect_worker_health "$health_output" disabled false docker_smoke_run "worker:start-db" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" docker run -d --name "$DB_CONTAINER_NAME" --network "$NETWORK_NAME" --network-alias db \ -e POSTGRES_USER="$DB_USER" \ -e POSTGRES_PASSWORD="$DB_PASSWORD" \ -e POSTGRES_DB="$DB_NAME" \ postgres:16-alpine >/dev/null docker_smoke_run "worker:start-redis" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" \ docker run -d --name "$REDIS_CONTAINER_NAME" --network "$NETWORK_NAME" --network-alias redis redis:7-alpine >/dev/null if ! docker image inspect "$MEILI_IMAGE" >/dev/null 2>&1; then docker_smoke_run "worker:pull-meili" "$MEILI_IMAGE_PULL_TIMEOUT_SECONDS" docker pull "$MEILI_IMAGE" >/dev/null fi docker_smoke_run "worker:start-meili" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" \ docker run -d --pull=never --name "$MEILI_CONTAINER_NAME" --network "$NETWORK_NAME" --network-alias meili \ -e MEILI_MASTER_KEY=docker-smoke-tenant-delete-meili-master-key \ -e MEILI_ENV=development \ -e MEILI_NO_ANALYTICS=true \ "$MEILI_IMAGE" >/dev/null MIGRATOR_DATABASE_URL="postgresql://hivegrid_migrator:${MIGRATOR_DB_PASSWORD}@db:5432/${DB_NAME}?schema=public" WORKER_DATABASE_URL="postgresql://hivegrid_worker:${WORKER_DB_PASSWORD}@db:5432/${DB_NAME}?schema=public" CONTAINER_REDIS_URL="redis://redis:6379/0" # Probe over TCP, not the unix socket. On first boot the postgres image starts a # temporary server that listens on the socket ONLY (initdb + init scripts), then # shuts it down before starting the real one. A socket-based pg_isready reports # ready during that window, so the very next psql can hit # "FATAL: the database system is shutting down". The temp server sets # listen_addresses='', so a TCP probe cannot succeed too early. if ! docker_smoke_wait_for_postgres "worker:probe-db" \ "$DB_CONTAINER_NAME" "$DB_USER" "$DB_NAME"; then echo "Postgres smoke database did not become ready within the ${DOCKER_SMOKE_DATABASE_READY_TIMEOUT_SECONDS}-second budget" >&2 docker_smoke_logs "worker:logs-db" "$DB_CONTAINER_NAME" >&2 || true exit 1 fi provision_docker_smoke_rls_roles \ "worker:provision-rls-roles" \ "$DB_CONTAINER_NAME" "$DB_USER" "$DB_NAME" \ "$MIGRATOR_DB_PASSWORD" "$APP_DB_PASSWORD" "$WORKER_DB_PASSWORD" "$AUTHENTIK_JIT_DB_PASSWORD" for attempt in $(seq 1 30); do if docker_smoke_probe "worker:probe-redis" docker exec "$REDIS_CONTAINER_NAME" redis-cli ping >/dev/null 2>&1; then break fi if [[ "$attempt" == "30" ]]; then echo "Redis smoke broker did not become ready" >&2 docker_smoke_logs "worker:logs-redis" "$REDIS_CONTAINER_NAME" >&2 || true exit 1 fi sleep 1 done for attempt in $(seq 1 60); do if docker_smoke_probe "worker:probe-meili" \ docker exec "$MEILI_CONTAINER_NAME" wget -q -O /dev/null http://127.0.0.1:7700/health; then break fi if [[ "$attempt" == "60" ]]; then echo "Meilisearch smoke service did not become ready" >&2 docker_smoke_logs "worker:logs-meili" "$MEILI_CONTAINER_NAME" >&2 || true exit 1 fi sleep 1 done # The web image owns the reviewed deployment migrations. Reuse that immutable # closure so the worker smoke needs neither host dependencies nor migration # tooling in the least-privilege worker runtime. docker_smoke_run "worker:migrate" "$DOCKER_SMOKE_MIGRATION_TIMEOUT_SECONDS" docker run --rm --name "$MIGRATION_CONTAINER_NAME" --network "$NETWORK_NAME" \ -e DIRECT_URL="$MIGRATOR_DATABASE_URL" \ "$MIGRATOR_IMAGE" migrate # Runtime workers consume the generated Prisma client but never migrate or # compile code. Guard the image-size and least-authority boundary directly. docker_smoke_run "worker:assert-runtime" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" docker run --rm --name "$ASSERT_CONTAINER_NAME" --entrypoint node "$IMAGE" -e ' const fs = require("node:fs"); const entries = fs.readdirSync("/myapp/node_modules/.pnpm"); const forbidden = [ /^prisma@/, /^typescript@/, /^@prisma\+engines@/, /^@prisma\+studio-core@/, /^@prisma\+dev@/, /^@electric-sql\+pglite(?:@|-)/, /^better-sqlite3@/, ]; const retained = entries.filter((entry) => forbidden.some((pattern) => pattern.test(entry))); if (retained.length > 0) { console.error(`Worker runtime retained build or migration toolchains: ${retained.join(", ")}`); process.exit(1); } ' >/dev/null enabled_health_output="" docker_smoke_capture enabled_health_output "worker:health-enabled" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" \ docker run --rm --name "$ENABLED_HEALTH_CONTAINER_NAME" --network "$NETWORK_NAME" \ "${worker_env[@]}" \ "${product_acquisition_env[@]}" \ "${general_hard_delete_env[@]}" \ -e WORKER_ENABLED=true \ -e WORKER_QUEUE_GROUP=general \ -e DATABASE_URL="$WORKER_DATABASE_URL" \ -e BULLMQ_REDIS_URL="$CONTAINER_REDIS_URL" \ "$IMAGE" node apps/worker/dist/health.js expect_worker_health "$enabled_health_output" ready true general hivegrid.notifications MISMATCHED_REVISION="$(printf '0%.0s' {1..40})" if [[ "$MISMATCHED_REVISION" == "$SMOKE_REVISION" ]]; then MISMATCHED_REVISION="$(printf 'f%.0s' {1..40})" fi mail_health_output="" docker_smoke_capture mail_health_output "worker:health-mail" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" \ docker run --rm --name "$MAIL_HEALTH_CONTAINER_NAME" --network "$NETWORK_NAME" \ "${worker_env[@]}" \ -e WORKER_ENABLED=true \ -e WORKER_QUEUE_GROUP=mail \ -e CRYPTO_SECRET=deploy-smoke-shared-tenant-envelope-secret-minimum-32 \ -e DATABASE_URL="$WORKER_DATABASE_URL" \ -e BULLMQ_REDIS_URL="$CONTAINER_REDIS_URL" \ "$IMAGE" node apps/worker/dist/health.js expect_worker_health "$mail_health_output" ready true mail hivegrid.email docker_smoke_run "worker:start-runtime" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" docker run -d --name "$CONTAINER_NAME" --network "$NETWORK_NAME" \ "${worker_env[@]}" \ "${product_acquisition_env[@]}" \ "${general_hard_delete_env[@]}" \ -e WORKER_ENABLED=true \ -e WORKER_QUEUE_GROUP=general \ -e DATABASE_URL="$WORKER_DATABASE_URL" \ -e BULLMQ_REDIS_URL="$CONTAINER_REDIS_URL" \ -e OUTBOX_DISPATCHER_ENABLED=true \ -e OUTBOX_DISPATCHER_REQUIRED=true \ -e WORKFLOW_TRIGGER_EXECUTOR_MODE=http \ -e WORKFLOW_TRIGGER_EXECUTOR_URL=http://127.0.0.1:9 \ -e API_ACCESS_TOKEN=deploy-smoke-worker-api-token \ -e WORKFLOW_TRIGGER_OUTBOX_ENABLED=true \ -e WORKFLOW_TRIGGER_OUTBOX_CONCURRENCY=5 \ -e WORKFLOW_TRIGGER_OUTBOX_REQUIRED=true \ "$IMAGE" >/dev/null for attempt in $(seq 1 10); do running="" docker_smoke_capture running "worker:inspect-running" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \ docker inspect -f '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null || true if [[ "$running" == "true" ]]; then break fi if [[ "$attempt" == "10" ]]; then echo "Worker container did not stay running" >&2 docker_smoke_logs "worker:logs-startup" "$CONTAINER_NAME" >&2 || true exit 1 fi sleep 1 done running_health_output="" docker_smoke_capture running_health_output "worker:health-runtime" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \ docker exec "$CONTAINER_NAME" node apps/worker/dist/health.js expect_worker_health "$running_health_output" ready true general hivegrid.notifications docker_smoke_run "worker:assert-revision-file" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" docker exec "$CONTAINER_NAME" sh -eu -c ' path=/myapp/.image-revision expected_revision="$1" owner_uid="$(stat -c "%u" "$path")" mode="$(stat -c "%a" "$path")" actual_revision="$(cat "$path")" if [ "$owner_uid" != "0" ] || [ $((0$mode & 022)) -ne 0 ] || [ "$actual_revision" != "$expected_revision" ]; then echo "Worker immutable image revision file is not root-owned, read-only, and exact" >&2 exit 1 fi ' _ "$SMOKE_REVISION" deploy_health_output="" for attempt in $(seq 1 20); do deploy_health_attempt="" if docker_smoke_capture deploy_health_attempt "worker:deploy-health" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \ bash -c 'docker exec "$@" 2>&1' _ "$CONTAINER_NAME" node apps/worker/dist/deploy-health.js "$SMOKE_REVISION" general 2>/dev/null; then deploy_health_output="$deploy_health_attempt" break fi if [[ "$attempt" == "20" ]]; then echo "Worker deployment health did not become ready" >&2 docker_smoke_run "worker:deploy-health-final" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \ docker exec "$CONTAINER_NAME" node apps/worker/dist/deploy-health.js "$SMOKE_REVISION" general >&2 || true docker_smoke_run "worker:readiness-diagnostic" "$DOCKER_SMOKE_LOG_TIMEOUT_SECONDS" docker exec "$CONTAINER_NAME" node -e ' fetch(`http://127.0.0.1:${process.env.WORKER_HEALTH_PORT || "9100"}/readyz`) .then(async (response) => process.stderr.write(`${response.status} ${await response.text()}\n`)) .catch(() => process.stderr.write("Worker readiness endpoint could not be queried\n")); ' >&2 || true docker_smoke_logs "worker:logs-deploy-health" "$CONTAINER_NAME" >&2 || true exit 1 fi sleep 1 done expect_worker_deploy_health "$deploy_health_output" general docker_smoke_expect_docker_exec_rejection "worker:reject-spoofed-revision" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" "$CONTAINER_NAME" \ env APP_REVISION="$MISMATCHED_REVISION" \ node apps/worker/dist/deploy-health.js "$MISMATCHED_REVISION" general docker_smoke_expect_docker_exec_rejection "worker:reject-wrong-group" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" "$CONTAINER_NAME" \ node apps/worker/dist/deploy-health.js "$SMOKE_REVISION" mail docker_smoke_expect_docker_exec_rejection "worker:reject-disabled" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" "$CONTAINER_NAME" \ env WORKER_ENABLED=false \ node apps/worker/dist/deploy-health.js "$SMOKE_REVISION" general WORKER_LOGS="" if ! WORKER_LOGS="$(docker_smoke_logs "worker:logs-deprecation" "$CONTAINER_NAME" 2>&1)"; then printf '%s\n' "$WORKER_LOGS" >&2 exit 1 fi if grep -q '\[DEP0169\]' <<<"$WORKER_LOGS"; then echo "Worker container used Node's deprecated url.parse() runtime path" >&2 docker_smoke_logs "worker:logs-deprecation-failure" "$CONTAINER_NAME" >&2 || true exit 1 fi echo "Docker worker runtime smoke passed for ${IMAGE}"