HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/scripts/docker-worker-runtime-smoke.sh

#!/usr/bin/env bash
set -euo pipefail
DOCKER_SMOKE_RUN_ID="${DOCKER_SMOKE_RUN_ID:-$$}"
if [[ -z "$DOCKER_SMOKE_RUN_ID" || ${#DOCKER_SMOKE_RUN_ID} -gt 80 || ! $DOCKER_SMOKE_RUN_ID =~ ^[A-Za-z0-9_.-]+$ ]]; then
  echo "DOCKER_SMOKE_RUN_ID must contain 1-80 Docker-safe characters" >&2
  exit 2
fi

# shellcheck source=scripts/lib/docker-rls-role-fixture.sh
source "$(dirname "${BASH_SOURCE[0]}")/lib/docker-rls-role-fixture.sh"
# shellcheck source=scripts/lib/docker-runtime-smoke.sh
source "$(dirname "${BASH_SOURCE[0]}")/lib/docker-runtime-smoke.sh"

IMAGE="${1:-hivegrid-worker:smoke}"
MIGRATOR_IMAGE="${DOCKER_SMOKE_MIGRATOR_IMAGE:-hivegrid-web:smoke}"
CONTAINER_NAME="${DOCKER_WORKER_SMOKE_CONTAINER:-hivegrid-worker-runtime-smoke-${DOCKER_SMOKE_RUN_ID}}"
DB_CONTAINER_NAME="${DOCKER_WORKER_SMOKE_DB_CONTAINER:-hivegrid-worker-runtime-smoke-db-${DOCKER_SMOKE_RUN_ID}}"
REDIS_CONTAINER_NAME="${DOCKER_WORKER_SMOKE_REDIS_CONTAINER:-hivegrid-worker-runtime-smoke-redis-${DOCKER_SMOKE_RUN_ID}}"
MEILI_CONTAINER_NAME="${DOCKER_WORKER_SMOKE_MEILI_CONTAINER:-hivegrid-worker-runtime-smoke-meili-${DOCKER_SMOKE_RUN_ID}}"
MEILI_IMAGE="${DOCKER_WORKER_SMOKE_MEILI_IMAGE:-getmeili/meilisearch:v1.8}"
MEILI_IMAGE_PULL_TIMEOUT_SECONDS="${DOCKER_WORKER_SMOKE_MEILI_IMAGE_PULL_TIMEOUT_SECONDS:-180}"
NETWORK_NAME="${DOCKER_WORKER_SMOKE_NETWORK:-hivegrid-worker-runtime-smoke-${DOCKER_SMOKE_RUN_ID}}"
MIGRATION_CONTAINER_NAME="${DOCKER_WORKER_SMOKE_MIGRATION_CONTAINER:-hivegrid-worker-runtime-smoke-migrate-${DOCKER_SMOKE_RUN_ID}}"
ASSERT_CONTAINER_NAME="${DOCKER_WORKER_SMOKE_ASSERT_CONTAINER:-hivegrid-worker-runtime-smoke-assert-${DOCKER_SMOKE_RUN_ID}}"
DISABLED_HEALTH_CONTAINER_NAME="${DOCKER_WORKER_SMOKE_DISABLED_HEALTH_CONTAINER:-hivegrid-worker-runtime-smoke-disabled-${DOCKER_SMOKE_RUN_ID}}"
ENABLED_HEALTH_CONTAINER_NAME="${DOCKER_WORKER_SMOKE_ENABLED_HEALTH_CONTAINER:-hivegrid-worker-runtime-smoke-enabled-${DOCKER_SMOKE_RUN_ID}}"
MAIL_HEALTH_CONTAINER_NAME="${DOCKER_WORKER_SMOKE_MAIL_HEALTH_CONTAINER:-hivegrid-worker-runtime-smoke-mail-${DOCKER_SMOKE_RUN_ID}}"
SMOKE_REVISION="${DOCKER_WORKER_SMOKE_REVISION:-}"
DB_PASSWORD="hivegrid"
DB_NAME="hivegrid"
DB_USER="hivegrid"
WORKER_DB_PASSWORD="hivegrid-worker-smoke"
APP_DB_PASSWORD="hivegrid-app-smoke"
MIGRATOR_DB_PASSWORD="hivegrid-migrator-smoke"
AUTHENTIK_JIT_DB_PASSWORD="hivegrid-authentik-jit-smoke"

cleanup() {
  docker_smoke_cleanup_container "worker:cleanup-containers" \
    "$CONTAINER_NAME" "$MIGRATION_CONTAINER_NAME" "$ASSERT_CONTAINER_NAME" \
    "$DISABLED_HEALTH_CONTAINER_NAME" "$ENABLED_HEALTH_CONTAINER_NAME" "$MAIL_HEALTH_CONTAINER_NAME" \
    "$DB_CONTAINER_NAME" "$REDIS_CONTAINER_NAME" "$MEILI_CONTAINER_NAME"
  docker_smoke_cleanup_network "worker:cleanup-network" "$NETWORK_NAME"
}

resolve_smoke_revision() {
  local image_revision

  if [[ -n "$SMOKE_REVISION" ]]; then
    printf "%s" "$SMOKE_REVISION"
    return
  fi

  docker_smoke_capture image_revision "worker:inspect-revision" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \
    docker image inspect -f '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$IMAGE"
  printf "%s" "$image_revision"
}

verify_migrator_revision() {
  local migrator_revision

  docker_smoke_capture migrator_revision "worker:inspect-migrator-revision" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \
    docker image inspect -f '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "$MIGRATOR_IMAGE"
  if [[ "$migrator_revision" != "$SMOKE_REVISION" ]]; then
    echo "Migrator image ${MIGRATOR_IMAGE} revision ${migrator_revision:-<empty>} does not match worker image revision ${SMOKE_REVISION}" >&2
    exit 1
  fi
}

docker_smoke_install_cleanup_traps cleanup
cleanup
SMOKE_REVISION="$(resolve_smoke_revision)"
if [[ -z "$SMOKE_REVISION" || "$SMOKE_REVISION" == "<no value>" || "$SMOKE_REVISION" == "unknown" ]]; then
  echo "Image ${IMAGE} does not expose a concrete org.opencontainers.image.revision; build with --build-arg APP_REVISION or set DOCKER_WORKER_SMOKE_REVISION explicitly" >&2
  exit 1
fi
verify_migrator_revision

worker_env=(
  -e NODE_ENV=production
  -e APP_NAME=HiveGrid
  -e APP_BASE_DOMAIN=localhost
  -e DATABASE_RUNTIME_ROLE=hivegrid_worker
  -e RLS_RUNTIME_MODE=required
  -e AUDIT_CHAIN_SECRET=deploy-smoke-audit-chain-secret-minimum-32
  -e ADMIN_CONNECTION_MASTER_KEY=deploy-smoke-admin-connection-key-minimum-32
)

# Obvious non-production fixtures prove that the required general-worker
# provider boundary is wired without storing any live credential in the repo.
product_acquisition_env=(
  -e LAGO_API_URL=http://lago.internal.test:3108
  -e LAGO_API_KEY=test-lago-runtime-smoke
  -e LAGO_TIMEOUT_MS=10000
  -e MOLLIE_API_KEY=test-mollie-runtime-smoke
  -e MOLLIE_PROFILE_ID=pfl_runtime_smoke
  -e BILLING_MOLLIE_WEBHOOK_URL=https://api.hivegrid.test/v1/billing/mollie/webhooks
  -e MOLLIE_TIMEOUT_MS=10000
)

# Obvious non-production fixtures keep the general-worker smoke aligned with
# its fail-closed export-retention and tenant-hard-delete contracts. The
# delete-only runtime must not receive the SSE-C key used by writers/readers.
general_hard_delete_env=(
  -e EXPORT_RETENTION_QUEUE_ENABLED=true
  -e STORAGE_ENDPOINT=https://storage.example.test
  -e STORAGE_REGION=eu-central-1
  -e STORAGE_BUCKET=docker-smoke-private-exports
  -e STORAGE_ACCESS_KEY_ID=docker-smoke-retention-access-key
  -e STORAGE_SECRET_ACCESS_KEY=docker-smoke-retention-secret-key
  -e STORAGE_FORCE_PATH_STYLE=false
  -e MEILI_URL=http://meili:7700
  -e MEILI_API_KEY=docker-smoke-tenant-delete-meili-master-key
)

expect_worker_health() {
  local output="$1"
  local expected_status="$2"
  local expected_enabled="$3"
  local expected_group="${4:-}"
  local expected_queue="${5:-}"

  node -e '
const [raw, expectedStatus, expectedEnabled, expectedRevision, expectedGroup, expectedQueue] = process.argv.slice(1);
const report = JSON.parse(raw);
if (report.status !== expectedStatus) {
  console.error(`Expected worker status ${expectedStatus}, got ${report.status}`);
  process.exit(1);
}

if (String(report.workerEnabled) !== expectedEnabled) {
  console.error(`Expected workerEnabled ${expectedEnabled}, got ${report.workerEnabled}`);
  process.exit(1);
}
if (report.revision !== expectedRevision) {
  console.error(`Expected revision ${expectedRevision}, got ${report.revision || "<empty>"}`);
  process.exit(1);
}
if (expectedGroup && report.workerGroup !== expectedGroup) {
  console.error(`Expected workerGroup ${expectedGroup}, got ${report.workerGroup || "<empty>"}`);
  process.exit(1);
}
if (expectedQueue && !Array.isArray(report.queues)) {
  console.error("Expected worker health queues to be an array");
  process.exit(1);
}
if (expectedQueue && !report.queues.includes(expectedQueue)) {
  console.error(`Expected worker queues to include ${expectedQueue}, got ${report.queues?.join(",") || "<empty>"}`);
  process.exit(1);
}
if (expectedStatus === "ready" && (report.databaseStatus !== "ready" || report.redisStatus !== "ready")) {
  console.error(`Expected ready DB/Redis, got database=${report.databaseStatus} redis=${report.redisStatus}`);
  process.exit(1);
}
' "$output" "$expected_status" "$expected_enabled" "$SMOKE_REVISION" "$expected_group" "$expected_queue"
}

expect_worker_deploy_health() {
  local output="$1"
  local expected_group="$2"

  node -e '
const [raw, expectedRevision, expectedGroup] = process.argv.slice(1);
const report = JSON.parse(raw);
if (report.status !== "ready" || report.revision !== expectedRevision || report.workerGroup !== expectedGroup) {
  console.error("Worker deployment health did not prove the expected revision and group");
  process.exit(1);
}
' "$output" "$SMOKE_REVISION" "$expected_group"
}

docker_smoke_run "worker:create-network" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" \
  docker network create "$NETWORK_NAME" >/dev/null

health_output=""
docker_smoke_capture health_output "worker:health-disabled" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" \
  docker run --rm --name "$DISABLED_HEALTH_CONTAINER_NAME" \
  "${worker_env[@]}" \
  -e WORKER_ENABLED=false \
  "$IMAGE" node apps/worker/dist/health.js

expect_worker_health "$health_output" disabled false

docker_smoke_run "worker:start-db" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" docker run -d --name "$DB_CONTAINER_NAME" --network "$NETWORK_NAME" --network-alias db \
  -e POSTGRES_USER="$DB_USER" \
  -e POSTGRES_PASSWORD="$DB_PASSWORD" \
  -e POSTGRES_DB="$DB_NAME" \
  postgres:16-alpine >/dev/null

docker_smoke_run "worker:start-redis" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" \
  docker run -d --name "$REDIS_CONTAINER_NAME" --network "$NETWORK_NAME" --network-alias redis redis:7-alpine >/dev/null

if ! docker image inspect "$MEILI_IMAGE" >/dev/null 2>&1; then
  docker_smoke_run "worker:pull-meili" "$MEILI_IMAGE_PULL_TIMEOUT_SECONDS" docker pull "$MEILI_IMAGE" >/dev/null
fi
docker_smoke_run "worker:start-meili" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" \
  docker run -d --pull=never --name "$MEILI_CONTAINER_NAME" --network "$NETWORK_NAME" --network-alias meili \
  -e MEILI_MASTER_KEY=docker-smoke-tenant-delete-meili-master-key \
  -e MEILI_ENV=development \
  -e MEILI_NO_ANALYTICS=true \
  "$MEILI_IMAGE" >/dev/null

MIGRATOR_DATABASE_URL="postgresql://hivegrid_migrator:${MIGRATOR_DB_PASSWORD}@db:5432/${DB_NAME}?schema=public"
WORKER_DATABASE_URL="postgresql://hivegrid_worker:${WORKER_DB_PASSWORD}@db:5432/${DB_NAME}?schema=public"
CONTAINER_REDIS_URL="redis://redis:6379/0"

# Probe over TCP, not the unix socket. On first boot the postgres image starts a
# temporary server that listens on the socket ONLY (initdb + init scripts), then
# shuts it down before starting the real one. A socket-based pg_isready reports
# ready during that window, so the very next psql can hit
# "FATAL: the database system is shutting down". The temp server sets
# listen_addresses='', so a TCP probe cannot succeed too early.
if ! docker_smoke_wait_for_postgres "worker:probe-db" \
  "$DB_CONTAINER_NAME" "$DB_USER" "$DB_NAME"; then
  echo "Postgres smoke database did not become ready within the ${DOCKER_SMOKE_DATABASE_READY_TIMEOUT_SECONDS}-second budget" >&2
  docker_smoke_logs "worker:logs-db" "$DB_CONTAINER_NAME" >&2 || true
  exit 1
fi

provision_docker_smoke_rls_roles \
  "worker:provision-rls-roles" \
  "$DB_CONTAINER_NAME" "$DB_USER" "$DB_NAME" \
  "$MIGRATOR_DB_PASSWORD" "$APP_DB_PASSWORD" "$WORKER_DB_PASSWORD" "$AUTHENTIK_JIT_DB_PASSWORD"

for attempt in $(seq 1 30); do
  if docker_smoke_probe "worker:probe-redis" docker exec "$REDIS_CONTAINER_NAME" redis-cli ping >/dev/null 2>&1; then
    break
  fi

  if [[ "$attempt" == "30" ]]; then
    echo "Redis smoke broker did not become ready" >&2
    docker_smoke_logs "worker:logs-redis" "$REDIS_CONTAINER_NAME" >&2 || true
    exit 1
  fi

  sleep 1
done

for attempt in $(seq 1 60); do
  if docker_smoke_probe "worker:probe-meili" \
    docker exec "$MEILI_CONTAINER_NAME" wget -q -O /dev/null http://127.0.0.1:7700/health; then
    break
  fi

  if [[ "$attempt" == "60" ]]; then
    echo "Meilisearch smoke service did not become ready" >&2
    docker_smoke_logs "worker:logs-meili" "$MEILI_CONTAINER_NAME" >&2 || true
    exit 1
  fi

  sleep 1
done

# The web image owns the reviewed deployment migrations. Reuse that immutable
# closure so the worker smoke needs neither host dependencies nor migration
# tooling in the least-privilege worker runtime.
docker_smoke_run "worker:migrate" "$DOCKER_SMOKE_MIGRATION_TIMEOUT_SECONDS" docker run --rm --name "$MIGRATION_CONTAINER_NAME" --network "$NETWORK_NAME" \
  -e DIRECT_URL="$MIGRATOR_DATABASE_URL" \
  "$MIGRATOR_IMAGE" migrate

# Runtime workers consume the generated Prisma client but never migrate or
# compile code. Guard the image-size and least-authority boundary directly.
docker_smoke_run "worker:assert-runtime" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" docker run --rm --name "$ASSERT_CONTAINER_NAME" --entrypoint node "$IMAGE" -e '
const fs = require("node:fs");
const entries = fs.readdirSync("/myapp/node_modules/.pnpm");
const forbidden = [
  /^prisma@/,
  /^typescript@/,
  /^@prisma\+engines@/,
  /^@prisma\+studio-core@/,
  /^@prisma\+dev@/,
  /^@electric-sql\+pglite(?:@|-)/,
  /^better-sqlite3@/,
];
const retained = entries.filter((entry) => forbidden.some((pattern) => pattern.test(entry)));
if (retained.length > 0) {
  console.error(`Worker runtime retained build or migration toolchains: ${retained.join(", ")}`);
  process.exit(1);
}
' >/dev/null

enabled_health_output=""
docker_smoke_capture enabled_health_output "worker:health-enabled" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" \
  docker run --rm --name "$ENABLED_HEALTH_CONTAINER_NAME" --network "$NETWORK_NAME" \
  "${worker_env[@]}" \
  "${product_acquisition_env[@]}" \
  "${general_hard_delete_env[@]}" \
  -e WORKER_ENABLED=true \
  -e WORKER_QUEUE_GROUP=general \
  -e DATABASE_URL="$WORKER_DATABASE_URL" \
  -e BULLMQ_REDIS_URL="$CONTAINER_REDIS_URL" \
  "$IMAGE" node apps/worker/dist/health.js
expect_worker_health "$enabled_health_output" ready true general hivegrid.notifications

MISMATCHED_REVISION="$(printf '0%.0s' {1..40})"
if [[ "$MISMATCHED_REVISION" == "$SMOKE_REVISION" ]]; then
  MISMATCHED_REVISION="$(printf 'f%.0s' {1..40})"
fi

mail_health_output=""
docker_smoke_capture mail_health_output "worker:health-mail" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" \
  docker run --rm --name "$MAIL_HEALTH_CONTAINER_NAME" --network "$NETWORK_NAME" \
  "${worker_env[@]}" \
  -e WORKER_ENABLED=true \
  -e WORKER_QUEUE_GROUP=mail \
  -e CRYPTO_SECRET=deploy-smoke-shared-tenant-envelope-secret-minimum-32 \
  -e DATABASE_URL="$WORKER_DATABASE_URL" \
  -e BULLMQ_REDIS_URL="$CONTAINER_REDIS_URL" \
  "$IMAGE" node apps/worker/dist/health.js
expect_worker_health "$mail_health_output" ready true mail hivegrid.email

docker_smoke_run "worker:start-runtime" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" docker run -d --name "$CONTAINER_NAME" --network "$NETWORK_NAME" \
  "${worker_env[@]}" \
  "${product_acquisition_env[@]}" \
  "${general_hard_delete_env[@]}" \
  -e WORKER_ENABLED=true \
  -e WORKER_QUEUE_GROUP=general \
  -e DATABASE_URL="$WORKER_DATABASE_URL" \
  -e BULLMQ_REDIS_URL="$CONTAINER_REDIS_URL" \
  -e OUTBOX_DISPATCHER_ENABLED=true \
  -e OUTBOX_DISPATCHER_REQUIRED=true \
  -e WORKFLOW_TRIGGER_EXECUTOR_MODE=http \
  -e WORKFLOW_TRIGGER_EXECUTOR_URL=http://127.0.0.1:9 \
  -e API_ACCESS_TOKEN=deploy-smoke-worker-api-token \
  -e WORKFLOW_TRIGGER_OUTBOX_ENABLED=true \
  -e WORKFLOW_TRIGGER_OUTBOX_CONCURRENCY=5 \
  -e WORKFLOW_TRIGGER_OUTBOX_REQUIRED=true \
  "$IMAGE" >/dev/null

for attempt in $(seq 1 10); do
  running=""
  docker_smoke_capture running "worker:inspect-running" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \
    docker inspect -f '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null || true
  if [[ "$running" == "true" ]]; then
    break
  fi

  if [[ "$attempt" == "10" ]]; then
    echo "Worker container did not stay running" >&2
    docker_smoke_logs "worker:logs-startup" "$CONTAINER_NAME" >&2 || true
    exit 1
  fi

  sleep 1
done

running_health_output=""
docker_smoke_capture running_health_output "worker:health-runtime" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \
  docker exec "$CONTAINER_NAME" node apps/worker/dist/health.js
expect_worker_health "$running_health_output" ready true general hivegrid.notifications

docker_smoke_run "worker:assert-revision-file" "$DOCKER_SMOKE_COMMAND_TIMEOUT_SECONDS" docker exec "$CONTAINER_NAME" sh -eu -c '
path=/myapp/.image-revision
expected_revision="$1"
owner_uid="$(stat -c "%u" "$path")"
mode="$(stat -c "%a" "$path")"
actual_revision="$(cat "$path")"
if [ "$owner_uid" != "0" ] || [ $((0$mode & 022)) -ne 0 ] || [ "$actual_revision" != "$expected_revision" ]; then
  echo "Worker immutable image revision file is not root-owned, read-only, and exact" >&2
  exit 1
fi
' _ "$SMOKE_REVISION"

deploy_health_output=""
for attempt in $(seq 1 20); do
  deploy_health_attempt=""
  if docker_smoke_capture deploy_health_attempt "worker:deploy-health" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \
    bash -c 'docker exec "$@" 2>&1' _ "$CONTAINER_NAME" node apps/worker/dist/deploy-health.js "$SMOKE_REVISION" general 2>/dev/null; then
    deploy_health_output="$deploy_health_attempt"
    break
  fi
  if [[ "$attempt" == "20" ]]; then
    echo "Worker deployment health did not become ready" >&2
    docker_smoke_run "worker:deploy-health-final" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" \
      docker exec "$CONTAINER_NAME" node apps/worker/dist/deploy-health.js "$SMOKE_REVISION" general >&2 || true
    docker_smoke_run "worker:readiness-diagnostic" "$DOCKER_SMOKE_LOG_TIMEOUT_SECONDS" docker exec "$CONTAINER_NAME" node -e '
fetch(`http://127.0.0.1:${process.env.WORKER_HEALTH_PORT || "9100"}/readyz`)
  .then(async (response) => process.stderr.write(`${response.status} ${await response.text()}\n`))
  .catch(() => process.stderr.write("Worker readiness endpoint could not be queried\n"));
' >&2 || true
    docker_smoke_logs "worker:logs-deploy-health" "$CONTAINER_NAME" >&2 || true
    exit 1
  fi
  sleep 1
done
expect_worker_deploy_health "$deploy_health_output" general

docker_smoke_expect_docker_exec_rejection "worker:reject-spoofed-revision" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" "$CONTAINER_NAME" \
  env APP_REVISION="$MISMATCHED_REVISION" \
  node apps/worker/dist/deploy-health.js "$MISMATCHED_REVISION" general
docker_smoke_expect_docker_exec_rejection "worker:reject-wrong-group" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" "$CONTAINER_NAME" \
  node apps/worker/dist/deploy-health.js "$SMOKE_REVISION" mail
docker_smoke_expect_docker_exec_rejection "worker:reject-disabled" "$DOCKER_SMOKE_PROBE_TIMEOUT_SECONDS" "$CONTAINER_NAME" \
  env WORKER_ENABLED=false \
  node apps/worker/dist/deploy-health.js "$SMOKE_REVISION" general

WORKER_LOGS=""
if ! WORKER_LOGS="$(docker_smoke_logs "worker:logs-deprecation" "$CONTAINER_NAME" 2>&1)"; then
  printf '%s\n' "$WORKER_LOGS" >&2
  exit 1
fi
if grep -q '\[DEP0169\]' <<<"$WORKER_LOGS"; then
  echo "Worker container used Node's deprecated url.parse() runtime path" >&2
  docker_smoke_logs "worker:logs-deprecation-failure" "$CONTAINER_NAME" >&2 || true
  exit 1
fi

echo "Docker worker runtime smoke passed for ${IMAGE}"