name: Deploy on: push: branches: [main] workflow_dispatch: inputs: action: description: "stage or promote" required: true default: "stage" staged_sha: description: "Full staged commit SHA to promote" required: false backup_confirmed: description: "Set to true only after production backup or restore-point readiness is confirmed for promote" required: false default: "false" env: TOOLCTRL_GITEA_HOST_IP: ${{ vars.TOOLCTRL_GITEA_HOST_IP }} concurrency: group: boilerplate-deploy cancel-in-progress: false jobs: deploy-preflight: name: Deploy preflight gates runs-on: toolctrl-ci timeout-minutes: 10 steps: - name: Resolve internal Gitea host run: | if [ -n "${TOOLCTRL_GITEA_HOST_IP:-}" ]; then if ! TOOLCTRL_GITEA_HOST_IP="$TOOLCTRL_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.TOOLCTRL_GITEA_HOST_IP || "") ? 0 : 1)'; then echo "TOOLCTRL_GITEA_HOST_IP must be a single IPv4 or IPv6 token" exit 1 fi printf '%s git.tool-ctrl.de\n' "$TOOLCTRL_GITEA_HOST_IP" >> /etc/hosts fi - name: Checkout code uses: actions/checkout@v4 - name: Verify deploy wiring run: | node scripts/verify-gitea-deploy-workflow.mjs node scripts/verify-deploy-readiness.mjs node scripts/verify-docker-baseline.mjs node scripts/verify-docker-non-root.mjs node --check scripts/coolify-deploy.mjs node --check scripts/coolify-ensure-worker-groups.mjs bash -n scripts/coolify-prepull-images.sh bash -n scripts/ci/install-cosign.sh stage-or-promote: name: Stage or promote Docker images runs-on: toolctrl-ci timeout-minutes: 60 needs: deploy-preflight env: REGISTRY: git.tool-ctrl.de IMAGE_NAMESPACE: toolctrl-labs IMAGE_PREFIX: hivegrid SHA: ${{ gitea.sha }} DISPATCH_ACTION: ${{ gitea.event.inputs.action }} DISPATCH_STAGED_SHA: ${{ gitea.event.inputs.staged_sha }} DISPATCH_BACKUP_CONFIRMED: ${{ gitea.event.inputs.backup_confirmed }} steps: - name: Resolve deployment action run: | REF="${GITEA_REF:-${{ gitea.ref }}}" EVENT_NAME="${GITEA_EVENT_NAME:-${{ gitea.event_name }}}" ACTION="${DISPATCH_ACTION:-stage}" STAGED_SHA="${DISPATCH_STAGED_SHA:-}" BACKUP_CONFIRMED="${DISPATCH_BACKUP_CONFIRMED:-false}" if [ "$EVENT_NAME" = "push" ] && [ "$REF" = "refs/heads/main" ]; then DEPLOY_ACTION=stage elif [ "$EVENT_NAME" = "workflow_dispatch" ] && [ "$ACTION" = "stage" ]; then if [ "$REF" != "refs/heads/main" ]; then echo "Stage must be dispatched from refs/heads/main" exit 1 fi DEPLOY_ACTION=stage elif [ "$EVENT_NAME" = "workflow_dispatch" ] && [ "$ACTION" = "promote" ]; then DEPLOY_ACTION=promote if [ "$REF" != "refs/heads/main" ]; then echo "Promote must be dispatched from refs/heads/main" exit 1 fi if ! printf '%s' "$STAGED_SHA" | grep -Eq '^[0-9a-f]{40}$'; then echo "Promote requires staged_sha as the full 40-character staged commit SHA" exit 1 fi if [ "$BACKUP_CONFIRMED" != "true" ]; then echo "Promote requires backup_confirmed=true after production backup or restore-point readiness is confirmed" exit 1 fi else echo "Unsupported deployment action: event=$EVENT_NAME ref=$REF action=$ACTION" exit 1 fi if ! printf '%s' "$SHA" | grep -Eq '^[0-9a-f]{40}$'; then echo "Gitea SHA must be a full 40-character commit SHA" exit 1 fi if [ "$DEPLOY_ACTION" = "promote" ]; then IMAGE_TAG="$STAGED_SHA" else IMAGE_TAG="$SHA" fi { echo "IMAGE_TAG=$IMAGE_TAG" echo "DEPLOY_ACTION=$DEPLOY_ACTION" } >> "$GITHUB_ENV" - name: Setup hosts override and Docker CLI run: | if [ -n "${TOOLCTRL_GITEA_HOST_IP:-}" ]; then if ! TOOLCTRL_GITEA_HOST_IP="$TOOLCTRL_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.TOOLCTRL_GITEA_HOST_IP || "") ? 0 : 1)'; then echo "TOOLCTRL_GITEA_HOST_IP must be a single IPv4 or IPv6 token" exit 1 fi printf '%s git.tool-ctrl.de\n' "$TOOLCTRL_GITEA_HOST_IP" >> /etc/hosts fi MISSING_TOOLS="" for TOOL in curl git; do if ! command -v "$TOOL" >/dev/null 2>&1; then MISSING_TOOLS="$MISSING_TOOLS $TOOL" fi done if ! command -v ssh >/dev/null 2>&1; then MISSING_TOOLS="$MISSING_TOOLS openssh-client" fi if [ -n "$MISSING_TOOLS" ]; then if ! command -v apt-get >/dev/null 2>&1 || [ "$(id -u)" != "0" ]; then echo "Missing required tools and apt-get is not available as root:$MISSING_TOOLS" exit 1 fi apt-get update -qq apt-get install -qq -y --no-install-recommends ca-certificates $MISSING_TOOLS fi if ! command -v docker >/dev/null 2>&1 || ! docker buildx version >/dev/null 2>&1; then if ! command -v apt-get >/dev/null 2>&1 || [ "$(id -u)" != "0" ]; then echo "Docker CLI/buildx missing and apt-get is not available as root" exit 1 fi install -m 0755 -d /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc chmod a+r /etc/apt/keyrings/docker.asc . /etc/os-release echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian ${VERSION_CODENAME} stable" > /etc/apt/sources.list.d/docker.list apt-get update -qq apt-get install -qq -y --no-install-recommends docker-ce-cli docker-buildx-plugin fi docker --version docker buildx version - name: Checkout code uses: actions/checkout@v4 with: fetch-depth: 0 - name: Verify checkout revision run: | CHECKOUT_SHA="$(git rev-parse HEAD)" if [ "$CHECKOUT_SHA" != "$SHA" ]; then echo "Checked out revision must match the Gitea SHA" exit 1 fi - name: Login to Gitea Registry run: echo "${{ secrets.REGISTRY_PAT }}" | docker login "$REGISTRY" --username "${{ secrets.REGISTRY_USER }}" --password-stdin - name: Verify staged promotion provenance run: | if [ "$DEPLOY_ACTION" != "promote" ]; then echo "Staged promotion provenance skipped for staging" exit 0 fi git fetch --no-tags origin main git cat-file -e "$IMAGE_TAG^{commit}" git merge-base --is-ancestor "$IMAGE_TAG" origin/main for TARGET in web api worker; do IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET" SHA_DIGEST="$(docker buildx imagetools inspect "$IMAGE:$IMAGE_TAG" --format '{{json .Manifest.Digest}}')" STAGING_DIGEST="$(docker buildx imagetools inspect "$IMAGE:staging" --format '{{json .Manifest.Digest}}')" if [ -z "$SHA_DIGEST" ] || [ "$SHA_DIGEST" != "$STAGING_DIGEST" ]; then echo "Staged image digest for $TARGET does not match $IMAGE_TAG" exit 1 fi done - name: Build and push staging images env: VITE_SENTRY_DSN_WEB: ${{ secrets.VITE_SENTRY_DSN_WEB }} VITE_SENTRY_ENVIRONMENT: ${{ secrets.VITE_SENTRY_ENVIRONMENT }} VITE_OTEL_EXPORTER_OTLP_ENDPOINT: ${{ secrets.VITE_OTEL_EXPORTER_OTLP_ENDPOINT }} run: | if [ "$DEPLOY_ACTION" != "stage" ]; then echo "Stage build skipped for promote" exit 0 fi cd "$GITHUB_WORKSPACE" BUILD_PIDS=() BUILD_TARGETS=() for TARGET in web api worker; do ( IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET" CACHE_REF="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/build-cache:$TARGET" WEB_BUILD_ARGS=() if [ "$TARGET" = "web" ]; then # VITE_* build args are only meaningful for the web target (Vite # inlines them into the client bundle). Keeping them web-only # prevents web env changes from invalidating api/worker caches. WEB_BUILD_ARGS=( --build-arg VITE_SENTRY_DSN_WEB="$VITE_SENTRY_DSN_WEB" --build-arg VITE_SENTRY_ENVIRONMENT="$VITE_SENTRY_ENVIRONMENT" --build-arg VITE_SENTRY_RELEASE="$SHA" --build-arg VITE_OTEL_EXPORTER_OTLP_ENDPOINT="$VITE_OTEL_EXPORTER_OTLP_ENDPOINT" --build-arg VITE_OTEL_SERVICE_NAME="boilerplate-web" ) fi # `ignore-error=true` on the cache export: writing the build cache # to the registry is an optimization, not part of producing/pushing # the image. The Gitea registry rate-limits (HTTP 429) build-cache # blob writes under bursts of back-to-back deploys (made worse by # web/api/worker exporting to the same cache in parallel); without # this flag a 429 on a cache layer fails the whole build even though # the image itself built and pushed fine. docker buildx build --push --target "$TARGET" --file Dockerfile \ --cache-from "type=registry,ref=$CACHE_REF" \ --cache-to "type=registry,ref=$CACHE_REF,mode=max,ignore-error=true" \ --build-arg APP_REVISION="$SHA" \ "${WEB_BUILD_ARGS[@]}" \ --tag "$IMAGE:$IMAGE_TAG" . ) & BUILD_PIDS+=("$!") BUILD_TARGETS+=("$TARGET") done BUILD_FAILED=0 for INDEX in "${!BUILD_PIDS[@]}"; do if ! wait "${BUILD_PIDS[$INDEX]}"; then echo "Docker build failed for ${BUILD_TARGETS[$INDEX]}" BUILD_FAILED=1 fi done if [ "$BUILD_FAILED" -ne 0 ]; then exit "$BUILD_FAILED" fi - name: Install cosign if: env.DEPLOY_ACTION == 'stage' run: bash scripts/ci/install-cosign.sh - name: Sign published container images if: env.DEPLOY_ACTION == 'stage' env: COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }} COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }} run: | if [ -z "${COSIGN_PRIVATE_KEY:-}" ] || [ -z "${COSIGN_PASSWORD:-}" ]; then echo "Cosign signing skipped: COSIGN_PRIVATE_KEY or COSIGN_PASSWORD not configured" exit 0 fi for TARGET in web api worker; do IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET" cosign sign --yes --key env://COSIGN_PRIVATE_KEY "$IMAGE:$IMAGE_TAG" done - name: Ensure Coolify staging worker groups env: COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }} COOLIFY_STAGING_RESOURCE_UUIDS: ${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }} COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }} COOLIFY_STAGING_PROVISION_WORKER_GROUPS: ${{ vars.COOLIFY_STAGING_PROVISION_WORKER_GROUPS }} COOLIFY_STAGING_PROJECT_UUID: ${{ vars.COOLIFY_STAGING_PROJECT_UUID }} COOLIFY_STAGING_SERVER_UUID: ${{ vars.COOLIFY_STAGING_SERVER_UUID }} COOLIFY_STAGING_ENVIRONMENT_NAME: ${{ vars.COOLIFY_STAGING_ENVIRONMENT_NAME }} COOLIFY_STAGING_ENVIRONMENT_UUID: ${{ vars.COOLIFY_STAGING_ENVIRONMENT_UUID }} COOLIFY_STAGING_DESTINATION_UUID: ${{ vars.COOLIFY_STAGING_DESTINATION_UUID }} COOLIFY_STAGING_WORKER_ENV_SOURCE_UUID: ${{ vars.COOLIFY_STAGING_WORKER_ENV_SOURCE_UUID }} run: | if [ "$DEPLOY_ACTION" != "stage" ]; then echo "Coolify staging worker group provisioning skipped for promote" exit 0 fi node scripts/coolify-ensure-worker-groups.mjs staging - name: Configure Coolify staging migration command env: COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }} COOLIFY_STAGING_RESOURCE_UUIDS: ${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }} COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }} MIGRATION_COMMAND: cd /myapp && pnpm exec prisma migrate deploy --config prisma.config.ts SEED_COMMAND: cd /myapp && SEED_DEMO_DATA=true ALLOW_STAGING_DEMO_SEED=true NODE_ENV=production node packages/db/dist/production-seed.server.js run: | if [ "$DEPLOY_ACTION" != "stage" ]; then echo "Coolify migration command configuration skipped for promote" exit 0 fi SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN" RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS="${STAGING_RESOLVED_RESOURCE_UUIDS:-$COOLIFY_STAGING_RESOURCE_UUIDS}" if [ -z "$SELECTED_COOLIFY_API_URL" ] || [ -z "$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" ] || [ -z "$SELECTED_COOLIFY_TOKEN" ]; then echo "Coolify staging configuration is incomplete" exit 1 fi WEB_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web" API_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" WORKER_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker" APPS_JSON="$(mktemp)" trap 'rm -f "${APPS_JSON:-}"' EXIT curl --fail --silent --show-error \ --header "Authorization: Bearer $SELECTED_COOLIFY_TOKEN" \ "$SELECTED_COOLIFY_API_URL/applications" > "$APPS_JSON" WEB_RESOURCE_UUID="$(WEB_IMAGE="$WEB_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.WEB_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")" API_RESOURCE_UUID="$(API_IMAGE="$API_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.API_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")" select_worker_resource() { WORKER_QUEUE_GROUP="$1" WORKER_IMAGE="$WORKER_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const group = process.env.WORKER_QUEUE_GROUP || ""; const envPattern = new RegExp(`WORKER_QUEUE_GROUP\\s*(?:=|:)\\s*["'\''"]?${group}\\b`, "i"); const namePattern = new RegExp(`(?:^|[\\s/_-])worker[\\s_-]*${group}(?:$|[\\s:_-])`, "i"); const app = apps.find((candidate) => { const text = [candidate.name, candidate.description, candidate.docker_registry_image_name, candidate.fqdn, candidate.domain, candidate.domains, JSON.stringify(candidate.environment_variables || candidate.envs || candidate.environment || "")].filter(Boolean).join(" "); return candidate.docker_registry_image_name === process.env.WORKER_IMAGE && allowed.has(candidate.uuid) && (envPattern.test(text) || namePattern.test(text)); }); process.stdout.write(app?.uuid || "");' "$APPS_JSON" } WORKER_GENERAL_RESOURCE_UUID="$(select_worker_resource general)" WORKER_MAIL_RESOURCE_UUID="$(select_worker_resource mail)" WORKER_IMPORT_RESOURCE_UUID="$(select_worker_resource import)" WORKER_WEBHOOK_RESOURCE_UUID="$(select_worker_resource webhook)" WORKER_PLUGIN_RESOURCE_UUID="$(select_worker_resource plugin)" WORKER_RESOURCE_UUIDS="$WORKER_GENERAL_RESOURCE_UUID,$WORKER_MAIL_RESOURCE_UUID,$WORKER_IMPORT_RESOURCE_UUID,$WORKER_WEBHOOK_RESOURCE_UUID,$WORKER_PLUGIN_RESOURCE_UUID" if [ -z "$WEB_RESOURCE_UUID" ] || [ -z "$API_RESOURCE_UUID" ] || [ -z "$WORKER_GENERAL_RESOURCE_UUID" ] || [ -z "$WORKER_MAIL_RESOURCE_UUID" ] || [ -z "$WORKER_IMPORT_RESOURCE_UUID" ] || [ -z "$WORKER_WEBHOOK_RESOURCE_UUID" ] || [ -z "$WORKER_PLUGIN_RESOURCE_UUID" ]; then echo "Coolify staging web, api, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications must all exist" exit 1 fi for REQUIRED_RESOURCE_UUID in "$WEB_RESOURCE_UUID" "$API_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do if ! printf ",%s," "$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" | grep -Fq ",$REQUIRED_RESOURCE_UUID,"; then echo "Coolify staging deploy resources must include the web, api, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications" exit 1 fi done DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS" { echo "STAGING_WEB_RESOURCE_UUID=$WEB_RESOURCE_UUID" echo "STAGING_WORKER_RESOURCE_UUIDS=$WORKER_RESOURCE_UUIDS" echo "STAGING_WORKER_GENERAL_RESOURCE_UUID=$WORKER_GENERAL_RESOURCE_UUID" echo "STAGING_WORKER_MAIL_RESOURCE_UUID=$WORKER_MAIL_RESOURCE_UUID" echo "STAGING_WORKER_IMPORT_RESOURCE_UUID=$WORKER_IMPORT_RESOURCE_UUID" echo "STAGING_WORKER_WEBHOOK_RESOURCE_UUID=$WORKER_WEBHOOK_RESOURCE_UUID" echo "STAGING_WORKER_PLUGIN_RESOURCE_UUID=$WORKER_PLUGIN_RESOURCE_UUID" echo "STAGING_DEPENDENT_RESOURCE_UUIDS=$DEPENDENT_RESOURCE_UUIDS" } >> "$GITHUB_ENV" MIGRATION_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ pre_deployment_command: process.env.MIGRATION_COMMAND, post_deployment_command: process.env.SEED_COMMAND, docker_registry_image_tag: process.env.IMAGE_TAG }))')" curl --fail --silent --show-error --request PATCH \ --header "Authorization: Bearer $SELECTED_COOLIFY_TOKEN" \ --header "Content-Type: application/json" \ --data "$MIGRATION_PAYLOAD" \ "$SELECTED_COOLIFY_API_URL/applications/$WEB_RESOURCE_UUID" >/dev/null IMAGE_TAG_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ docker_registry_image_tag: process.env.IMAGE_TAG }))')" for RESOURCE_UUID in "$API_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do curl --fail --silent --show-error --request PATCH \ --header "Authorization: Bearer $SELECTED_COOLIFY_TOKEN" \ --header "Content-Type: application/json" \ --data "$IMAGE_TAG_PAYLOAD" \ "$SELECTED_COOLIFY_API_URL/applications/$RESOURCE_UUID" >/dev/null done - name: Publish staging image aliases run: | if [ "$DEPLOY_ACTION" != "stage" ]; then echo "Staging alias publication skipped for promote" exit 0 fi for TARGET in web api worker; do IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET" docker buildx imagetools create --tag "$IMAGE:staging" "$IMAGE:$IMAGE_TAG" done - name: Pre-pull staging images on Coolify host env: COOLIFY_STAGING_SSH_HOST: ${{ vars.COOLIFY_STAGING_SSH_HOST }} COOLIFY_STAGING_SSH_USER: ${{ vars.COOLIFY_STAGING_SSH_USER }} COOLIFY_STAGING_SSH_KEY: ${{ secrets.COOLIFY_STAGING_SSH_KEY }} REGISTRY_USER: ${{ secrets.REGISTRY_USER }} REGISTRY_PAT: ${{ secrets.REGISTRY_PAT }} run: | if [ "$DEPLOY_ACTION" != "stage" ]; then echo "Staging image pre-pull skipped for promote" exit 0 fi timeout 240s scripts/coolify-prepull-images.sh || echo "Staging image pre-pull skipped after timeout; Coolify will pull during deploy" - name: Trigger Coolify staging deploy env: COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }} COOLIFY_STAGING_RESOURCE_UUIDS: ${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }} COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }} run: | if [ "$DEPLOY_ACTION" != "stage" ]; then echo "Staging deploy skipped for promote" exit 0 fi SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN" if [ -z "$SELECTED_COOLIFY_API_URL" ] || [ -z "$STAGING_WEB_RESOURCE_UUID" ] || [ -z "$SELECTED_COOLIFY_TOKEN" ]; then echo "Coolify staging configuration is incomplete" exit 1 fi STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")" echo "STAGING_WEB_DEPLOYMENT_UUID=$STAGING_WEB_DEPLOYMENT_UUID" >> "$GITHUB_ENV" - name: Wait for Coolify staging health env: COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }} COOLIFY_STAGING_RESOURCE_UUIDS: ${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }} COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }} run: | if [ "$DEPLOY_ACTION" != "stage" ]; then echo "Staging health wait skipped for promote" exit 0 fi SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN" if [ -z "$SELECTED_COOLIFY_API_URL" ] || [ -z "$STAGING_WEB_RESOURCE_UUID" ] || [ -z "$SELECTED_COOLIFY_TOKEN" ]; then echo "Coolify staging configuration is incomplete" exit 1 fi if [ -z "$STAGING_WEB_DEPLOYMENT_UUID" ]; then echo "Coolify staging deployment UUID is missing" exit 1 fi node scripts/coolify-deploy.mjs wait "$STAGING_WEB_DEPLOYMENT_UUID" "Coolify staging web" verify_live_revision() { APP_JSON="$1" LABEL="$2" REQUIRE_HEALTH_URL="${3:-auto}" EXPECTED_IMAGE_ALIAS="${4:-staging}" EXPECTED_SERVICE="${5:-}" HEALTH_URL="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const raw = app.fqdn || app.domain || app.domains || ""; const value = Array.isArray(raw) ? raw[0] : String(raw).split(/[,\s]+/).find(Boolean); process.stdout.write(value ? value.replace(/\/+$/, "") : "");' "$APP_JSON")" HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")" if [ -z "$HEALTH_URL" ]; then if [ "$REQUIRE_HEALTH_URL" = "required" ] || [ "$HTTP_RESOURCE" = "true" ]; then echo "$LABEL is an HTTP resource but has no health URL" exit 1 fi CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")" if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then echo "$LABEL non-HTTP resource image tag does not match expected alias" exit 1 fi echo "$LABEL non-HTTP resource image tag matches expected alias" return fi HEALTH_JSON="$(mktemp)" if ! curl --fail --silent --show-error "$HEALTH_URL/healthz" > "$HEALTH_JSON"; then rm -f "$HEALTH_JSON" echo "$LABEL health endpoint is not ready yet" return 1 fi HEALTH_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$HEALTH_JSON")" HEALTH_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$HEALTH_JSON")" rm -f "$HEALTH_JSON" if [ -n "$EXPECTED_SERVICE" ] && [ "$HEALTH_SERVICE" != "$EXPECTED_SERVICE" ]; then echo "$LABEL health service does not match expected service" exit 1 fi if [ "$HEALTH_REVISION" != "$IMAGE_TAG" ]; then echo "$LABEL health revision does not match staged image yet" return 1 fi } RESOURCE_INDEX=0 for RESOURCE_UUID in $(printf "%s" "$STAGING_WEB_RESOURCE_UUID" | tr ',' ' '); do RESOURCE_INDEX=$((RESOURCE_INDEX + 1)) HEALTHY=false for ATTEMPT in $(seq 1 24); do APP_JSON="$(mktemp)" curl --fail --silent --show-error \ --header "Authorization: Bearer $SELECTED_COOLIFY_TOKEN" \ "$SELECTED_COOLIFY_API_URL/applications/$RESOURCE_UUID" > "$APP_JSON" STATUS="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.status || "unknown");' "$APP_JSON")" echo "Coolify staging resource #$RESOURCE_INDEX status: $STATUS" if [ "$STATUS" = "running:healthy" ]; then if verify_live_revision "$APP_JSON" "Coolify staging resource #$RESOURCE_INDEX" required "$IMAGE_TAG" web; then rm -f "$APP_JSON" HEALTHY=true break fi fi rm -f "$APP_JSON" if printf "%s" "$STATUS" | grep -Eiq 'failed|exited|dead'; then echo "Coolify staging resource #$RESOURCE_INDEX entered a terminal failure status" exit 1 fi sleep 5 done if [ "$HEALTHY" != "true" ]; then echo "Coolify staging resource #$RESOURCE_INDEX did not become healthy in time" exit 1 fi done - name: Trigger Coolify staging dependent deploys env: COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }} COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }} run: | if [ "$DEPLOY_ACTION" != "stage" ]; then echo "Staging dependent deploy skipped for promote" exit 0 fi if [ -z "$STAGING_DEPENDENT_RESOURCE_UUIDS" ]; then echo "No staging dependent resources to deploy" exit 0 fi SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN" if [ -z "$SELECTED_COOLIFY_API_URL" ] || [ -z "$SELECTED_COOLIFY_TOKEN" ]; then echo "Coolify staging configuration is incomplete" exit 1 fi STAGING_DEPENDENT_DEPLOYMENT_UUIDS="" for RESOURCE_UUID in $(printf "%s" "$STAGING_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$RESOURCE_UUID")" STAGING_DEPENDENT_DEPLOYMENT_UUIDS="${STAGING_DEPENDENT_DEPLOYMENT_UUIDS:+$STAGING_DEPENDENT_DEPLOYMENT_UUIDS,}$DEPLOYMENT_UUID" done echo "STAGING_DEPENDENT_DEPLOYMENT_UUIDS=$STAGING_DEPENDENT_DEPLOYMENT_UUIDS" >> "$GITHUB_ENV" - name: Wait for Coolify staging dependent health env: COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }} COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }} COOLIFY_STAGING_SSH_HOST: ${{ vars.COOLIFY_STAGING_SSH_HOST }} COOLIFY_STAGING_SSH_USER: ${{ vars.COOLIFY_STAGING_SSH_USER }} COOLIFY_STAGING_SSH_KEY: ${{ secrets.COOLIFY_STAGING_SSH_KEY }} run: | if [ "$DEPLOY_ACTION" != "stage" ]; then echo "Staging dependent health wait skipped for promote" exit 0 fi if [ -z "$STAGING_DEPENDENT_RESOURCE_UUIDS" ]; then echo "No staging dependent resources to wait for" exit 0 fi if [ -z "$STAGING_DEPENDENT_DEPLOYMENT_UUIDS" ]; then echo "Coolify staging dependent deployment UUIDs are missing" exit 1 fi SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN" if [ -z "$SELECTED_COOLIFY_API_URL" ] || [ -z "$SELECTED_COOLIFY_TOKEN" ]; then echo "Coolify staging configuration is incomplete" exit 1 fi # SSH helper for worker health verification (Coolify has no exec REST API); # used by verify_live_revision to run the health probe via docker exec. if [ -z "$COOLIFY_STAGING_SSH_HOST" ] || [ -z "$COOLIFY_STAGING_SSH_KEY" ]; then echo "Coolify staging SSH configuration is incomplete (worker health verification requires SSH + docker exec)" exit 1 fi REMOTE_SSH_KEY_FILE="$(mktemp)" REMOTE_SSH_KNOWN_HOSTS="$(mktemp)" trap 'rm -f "$REMOTE_SSH_KEY_FILE" "$REMOTE_SSH_KNOWN_HOSTS"' EXIT # Materialise the key via an intermediate var so the secret is written to # a file (never stdout/logs); avoids leaking it in CI output. REMOTE_SSH_KEY_DATA="$COOLIFY_STAGING_SSH_KEY" printf '%s\n' "$REMOTE_SSH_KEY_DATA" > "$REMOTE_SSH_KEY_FILE" unset REMOTE_SSH_KEY_DATA chmod 600 "$REMOTE_SSH_KEY_FILE" REMOTE_SSH_USER="${COOLIFY_STAGING_SSH_USER:-root}" REMOTE_SSH_HOST="$COOLIFY_STAGING_SSH_HOST" if command -v ssh-keyscan >/dev/null 2>&1; then ssh-keyscan -H "$REMOTE_SSH_HOST" > "$REMOTE_SSH_KNOWN_HOSTS" 2>/dev/null || true fi if [ -s "$REMOTE_SSH_KNOWN_HOSTS" ]; then REMOTE_SSH_HOSTKEY_OPT="-o StrictHostKeyChecking=yes" else REMOTE_SSH_HOSTKEY_OPT="-o StrictHostKeyChecking=accept-new" fi remote_ssh() { ssh -i "$REMOTE_SSH_KEY_FILE" -o BatchMode=yes -o ConnectTimeout=10 -o UserKnownHostsFile="$REMOTE_SSH_KNOWN_HOSTS" $REMOTE_SSH_HOSTKEY_OPT "${REMOTE_SSH_USER}@${REMOTE_SSH_HOST}" "$@" } DEPLOYMENT_INDEX=0 DEPLOYMENT_WAIT_PIDS="" for DEPLOYMENT_UUID in $(printf "%s" "$STAGING_DEPENDENT_DEPLOYMENT_UUIDS" | tr ',' ' '); do DEPLOYMENT_INDEX=$((DEPLOYMENT_INDEX + 1)) node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify staging dependent #$DEPLOYMENT_INDEX" & DEPLOYMENT_WAIT_PIDS="$DEPLOYMENT_WAIT_PIDS $!" done DEPLOYMENT_WAIT_FAILED=false for PID in $DEPLOYMENT_WAIT_PIDS; do if ! wait "$PID"; then DEPLOYMENT_WAIT_FAILED=true fi done if [ "$DEPLOYMENT_WAIT_FAILED" = "true" ]; then echo "One or more Coolify staging dependent deployment waits failed" exit 1 fi verify_live_revision() { APP_JSON="$1" LABEL="$2" REQUIRE_HEALTH_URL="${3:-auto}" EXPECTED_IMAGE_ALIAS="${4:-staging}" EXPECTED_SERVICE="${5:-}" RESOURCE_KIND="${6:-auto}" EXPECTED_WORKER_GROUP="${7:-}" HEALTH_URL="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const raw = app.fqdn || app.domain || app.domains || ""; const value = Array.isArray(raw) ? raw[0] : String(raw).split(/[,\s]+/).find(Boolean); process.stdout.write(value ? value.replace(/\/+$/, "") : "");' "$APP_JSON")" HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")" WORKER_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])worker(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")" if [ "$RESOURCE_KIND" = "worker" ] || [ "$WORKER_RESOURCE" = "true" ]; then CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")" if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then echo "$LABEL worker image tag does not match expected alias" exit 1 fi # Coolify 4.x exposes no command-execution REST API, so verify worker # health by running the one-shot health probe inside the live container # over SSH (docker exec) on the Coolify host. Fail-closed: any SSH / # container-resolution / exec failure blocks the deploy. WORKER_CONTAINER="$(remote_ssh "docker ps --filter name=$RESOURCE_UUID --filter status=running --format '{{.Names}}' | head -n1" 2>/dev/null | tr -d '\r')" if [ -z "$WORKER_CONTAINER" ]; then echo "$LABEL worker container not found on Coolify host" exit 1 fi # health.js exits non-zero when the worker reports a non-ready # status (e.g. "misconfigured"), yet it still prints the diagnostic # JSON (status + errors) to stdout. Capture stdout regardless of the # exit code (|| true) so the real reason surfaces in the checks below # instead of a generic "command unavailable". Empty output means the # docker exec itself could not run (container gone, SSH failure, ...). WORKER_HEALTH_OUTPUT="$(remote_ssh "docker exec $WORKER_CONTAINER node apps/worker/dist/health.js" 2>/dev/null | tr -d '\r' || true)" if [ -z "$WORKER_HEALTH_OUTPUT" ]; then echo "$LABEL worker docker exec health command unavailable (no output from health probe)" exit 1 fi WORKER_HEALTH_JSON="$(mktemp)" printf "%s" "$WORKER_HEALTH_OUTPUT" > "$WORKER_HEALTH_JSON" if ! WORKER_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$WORKER_HEALTH_JSON" 2>/dev/null)"; then rm -f "$WORKER_HEALTH_JSON" echo "$LABEL worker execute output is not JSON" exit 1 fi WORKER_STATUS="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.status || "");' "$WORKER_HEALTH_JSON")" WORKER_ENABLED="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.workerEnabled === true ? "true" : "false");' "$WORKER_HEALTH_JSON")" WORKER_GROUP="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.workerGroup || "");' "$WORKER_HEALTH_JSON")" WORKER_QUEUES="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(Array.isArray(payload.queues) ? payload.queues.join(",") : "");' "$WORKER_HEALTH_JSON")" WORKER_HEALTH_ERRORS="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(Array.isArray(payload.errors) ? payload.errors.join("; ") : "");' "$WORKER_HEALTH_JSON")" rm -f "$WORKER_HEALTH_JSON" if [ "$WORKER_REVISION" != "$IMAGE_TAG" ]; then echo "$LABEL worker health revision does not match image tag yet" return 1 fi if [ "$WORKER_ENABLED" != "true" ]; then echo "$LABEL worker health reports workerEnabled=false" exit 1 fi if [ "$WORKER_STATUS" != "ready" ]; then echo "$LABEL worker health status is not ready: '$WORKER_STATUS'${WORKER_HEALTH_ERRORS:+ (errors: $WORKER_HEALTH_ERRORS)}" exit 1 fi if [ -n "$EXPECTED_WORKER_GROUP" ] && [ "$WORKER_GROUP" != "$EXPECTED_WORKER_GROUP" ]; then echo "$LABEL worker group does not match expected queue group" exit 1 fi EXPECTED_WORKER_QUEUES="" case "$EXPECTED_WORKER_GROUP" in general) EXPECTED_WORKER_QUEUES="toolctrl.notifications toolctrl.maintenance toolctrl.file-scan" ;; mail) EXPECTED_WORKER_QUEUES="toolctrl.email" ;; import) if [ -n "$WORKER_QUEUES" ]; then EXPECTED_WORKER_QUEUES="toolctrl.imports toolctrl.exports" fi ;; webhook) EXPECTED_WORKER_QUEUES="toolctrl.ai-chatbot toolctrl.browser-automation" ;; plugin) EXPECTED_WORKER_QUEUES="toolctrl.plugin-invoke" ;; esac for EXPECTED_WORKER_QUEUE in $EXPECTED_WORKER_QUEUES; do if ! printf ",%s," "$WORKER_QUEUES" | grep -Fq ",$EXPECTED_WORKER_QUEUE,"; then echo "$LABEL worker queues do not match expected queue group" exit 1 fi done echo "$LABEL worker live revision matches image tag" return fi if [ -z "$HEALTH_URL" ]; then if [ "$REQUIRE_HEALTH_URL" = "required" ] || [ "$HTTP_RESOURCE" = "true" ]; then echo "$LABEL is an HTTP resource but has no health URL" exit 1 fi CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")" if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then echo "$LABEL non-HTTP resource image tag does not match expected alias" exit 1 fi echo "$LABEL non-HTTP resource image tag matches expected alias" return fi HEALTH_JSON="$(mktemp)" if ! curl --fail --silent --show-error "$HEALTH_URL/healthz" > "$HEALTH_JSON"; then rm -f "$HEALTH_JSON" echo "$LABEL health endpoint is not ready yet" return 1 fi HEALTH_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$HEALTH_JSON")" HEALTH_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$HEALTH_JSON")" rm -f "$HEALTH_JSON" if [ -n "$EXPECTED_SERVICE" ] && [ "$HEALTH_SERVICE" != "$EXPECTED_SERVICE" ]; then echo "$LABEL health service does not match expected service" exit 1 fi if [ "$HEALTH_REVISION" != "$IMAGE_TAG" ]; then echo "$LABEL health revision does not match staged image yet" return 1 fi } check_dependent_resource() { RESOURCE_UUID="$1" RESOURCE_INDEX="$2" HEALTHY=false for ATTEMPT in $(seq 1 24); do APP_JSON="$(mktemp)" curl --fail --silent --show-error \ --header "Authorization: Bearer $SELECTED_COOLIFY_TOKEN" \ "$SELECTED_COOLIFY_API_URL/applications/$RESOURCE_UUID" > "$APP_JSON" STATUS="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.status || "unknown");' "$APP_JSON")" echo "Coolify staging dependent resource #$RESOURCE_INDEX status: $STATUS" if [ "$STATUS" = "running:healthy" ]; then RESOURCE_KIND="api" EXPECTED_SERVICE="api" EXPECTED_WORKER_GROUP="" if printf ",%s," "$STAGING_WORKER_RESOURCE_UUIDS" | grep -Fq ",$RESOURCE_UUID,"; then RESOURCE_KIND="worker" EXPECTED_SERVICE="" case "$RESOURCE_UUID" in "$STAGING_WORKER_GENERAL_RESOURCE_UUID") EXPECTED_WORKER_GROUP="general" ;; "$STAGING_WORKER_MAIL_RESOURCE_UUID") EXPECTED_WORKER_GROUP="mail" ;; "$STAGING_WORKER_IMPORT_RESOURCE_UUID") EXPECTED_WORKER_GROUP="import" ;; "$STAGING_WORKER_WEBHOOK_RESOURCE_UUID") EXPECTED_WORKER_GROUP="webhook" ;; "$STAGING_WORKER_PLUGIN_RESOURCE_UUID") EXPECTED_WORKER_GROUP="plugin" ;; esac fi if verify_live_revision "$APP_JSON" "Coolify staging dependent resource #$RESOURCE_INDEX" auto "$IMAGE_TAG" "$EXPECTED_SERVICE" "$RESOURCE_KIND" "$EXPECTED_WORKER_GROUP"; then rm -f "$APP_JSON" HEALTHY=true break fi fi rm -f "$APP_JSON" if printf "%s" "$STATUS" | grep -Eiq 'failed|exited|dead'; then echo "Coolify staging dependent resource #$RESOURCE_INDEX entered a terminal failure status" exit 1 fi sleep 5 done if [ "$HEALTHY" != "true" ]; then echo "Coolify staging dependent resource #$RESOURCE_INDEX did not become healthy in time" exit 1 fi } RESOURCE_INDEX=0 RESOURCE_WAIT_PIDS="" for RESOURCE_UUID in $(printf "%s" "$STAGING_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do RESOURCE_INDEX=$((RESOURCE_INDEX + 1)) check_dependent_resource "$RESOURCE_UUID" "$RESOURCE_INDEX" & RESOURCE_WAIT_PIDS="$RESOURCE_WAIT_PIDS $!" done RESOURCE_WAIT_FAILED=false for PID in $RESOURCE_WAIT_PIDS; do if ! wait "$PID"; then RESOURCE_WAIT_FAILED=true fi done if [ "$RESOURCE_WAIT_FAILED" = "true" ]; then echo "One or more Coolify staging dependent resources failed health checks" exit 1 fi - name: Ensure Coolify production worker groups env: COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }} COOLIFY_RESOURCE_UUIDS: ${{ vars.COOLIFY_RESOURCE_UUIDS }} COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }} COOLIFY_PROVISION_WORKER_GROUPS: ${{ vars.COOLIFY_PROVISION_WORKER_GROUPS }} COOLIFY_PROJECT_UUID: ${{ vars.COOLIFY_PROJECT_UUID }} COOLIFY_SERVER_UUID: ${{ vars.COOLIFY_SERVER_UUID }} COOLIFY_ENVIRONMENT_NAME: ${{ vars.COOLIFY_ENVIRONMENT_NAME }} COOLIFY_ENVIRONMENT_UUID: ${{ vars.COOLIFY_ENVIRONMENT_UUID }} COOLIFY_DESTINATION_UUID: ${{ vars.COOLIFY_DESTINATION_UUID }} COOLIFY_WORKER_ENV_SOURCE_UUID: ${{ vars.COOLIFY_WORKER_ENV_SOURCE_UUID }} run: | if [ "$DEPLOY_ACTION" != "promote" ]; then echo "Coolify production worker group provisioning skipped for staging" exit 0 fi node scripts/coolify-ensure-worker-groups.mjs production - name: Configure Coolify production migration command env: COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }} COOLIFY_RESOURCE_UUIDS: ${{ vars.COOLIFY_RESOURCE_UUIDS }} COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }} MIGRATION_COMMAND: cd /myapp && pnpm exec prisma migrate deploy --config prisma.config.ts SEED_COMMAND: cd /myapp && NODE_ENV=production node packages/db/dist/production-seed.server.js run: | if [ "$DEPLOY_ACTION" != "promote" ]; then echo "Coolify production migration command configuration skipped for staging" exit 0 fi RESOLVED_COOLIFY_RESOURCE_UUIDS="${PRODUCTION_RESOLVED_RESOURCE_UUIDS:-$COOLIFY_RESOURCE_UUIDS}" if [ -z "$COOLIFY_API_URL" ] || [ -z "$RESOLVED_COOLIFY_RESOURCE_UUIDS" ] || [ -z "$COOLIFY_TOKEN" ]; then echo "Coolify production configuration is incomplete" exit 1 fi WEB_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web" API_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" WORKER_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker" APPS_JSON="$(mktemp)" trap 'rm -f "${APPS_JSON:-}"' EXIT curl --fail --silent --show-error \ --header "Authorization: Bearer $COOLIFY_TOKEN" \ "$COOLIFY_API_URL/applications" > "$APPS_JSON" WEB_RESOURCE_UUID="$(WEB_IMAGE="$WEB_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.WEB_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")" API_RESOURCE_UUID="$(API_IMAGE="$API_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.API_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")" select_worker_resource() { WORKER_QUEUE_GROUP="$1" WORKER_IMAGE="$WORKER_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const group = process.env.WORKER_QUEUE_GROUP || ""; const envPattern = new RegExp(`WORKER_QUEUE_GROUP\\s*(?:=|:)\\s*["'\''"]?${group}\\b`, "i"); const namePattern = new RegExp(`(?:^|[\\s/_-])worker[\\s_-]*${group}(?:$|[\\s:_-])`, "i"); const app = apps.find((candidate) => { const text = [candidate.name, candidate.description, candidate.docker_registry_image_name, candidate.fqdn, candidate.domain, candidate.domains, JSON.stringify(candidate.environment_variables || candidate.envs || candidate.environment || "")].filter(Boolean).join(" "); return candidate.docker_registry_image_name === process.env.WORKER_IMAGE && allowed.has(candidate.uuid) && (envPattern.test(text) || namePattern.test(text)); }); process.stdout.write(app?.uuid || "");' "$APPS_JSON" } WORKER_GENERAL_RESOURCE_UUID="$(select_worker_resource general)" WORKER_MAIL_RESOURCE_UUID="$(select_worker_resource mail)" WORKER_IMPORT_RESOURCE_UUID="$(select_worker_resource import)" WORKER_WEBHOOK_RESOURCE_UUID="$(select_worker_resource webhook)" WORKER_PLUGIN_RESOURCE_UUID="$(select_worker_resource plugin)" WORKER_RESOURCE_UUIDS="$WORKER_GENERAL_RESOURCE_UUID,$WORKER_MAIL_RESOURCE_UUID,$WORKER_IMPORT_RESOURCE_UUID,$WORKER_WEBHOOK_RESOURCE_UUID,$WORKER_PLUGIN_RESOURCE_UUID" if [ -z "$WEB_RESOURCE_UUID" ] || [ -z "$API_RESOURCE_UUID" ] || [ -z "$WORKER_GENERAL_RESOURCE_UUID" ] || [ -z "$WORKER_MAIL_RESOURCE_UUID" ] || [ -z "$WORKER_IMPORT_RESOURCE_UUID" ] || [ -z "$WORKER_WEBHOOK_RESOURCE_UUID" ] || [ -z "$WORKER_PLUGIN_RESOURCE_UUID" ]; then echo "Coolify production web, api, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications must all exist" exit 1 fi for REQUIRED_RESOURCE_UUID in "$WEB_RESOURCE_UUID" "$API_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do if ! printf ",%s," "$RESOLVED_COOLIFY_RESOURCE_UUIDS" | grep -Fq ",$REQUIRED_RESOURCE_UUID,"; then echo "Coolify production deploy resources must include the web, api, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications" exit 1 fi done DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS" { echo "PRODUCTION_WEB_RESOURCE_UUID=$WEB_RESOURCE_UUID" echo "PRODUCTION_WORKER_RESOURCE_UUIDS=$WORKER_RESOURCE_UUIDS" echo "PRODUCTION_WORKER_GENERAL_RESOURCE_UUID=$WORKER_GENERAL_RESOURCE_UUID" echo "PRODUCTION_WORKER_MAIL_RESOURCE_UUID=$WORKER_MAIL_RESOURCE_UUID" echo "PRODUCTION_WORKER_IMPORT_RESOURCE_UUID=$WORKER_IMPORT_RESOURCE_UUID" echo "PRODUCTION_WORKER_WEBHOOK_RESOURCE_UUID=$WORKER_WEBHOOK_RESOURCE_UUID" echo "PRODUCTION_WORKER_PLUGIN_RESOURCE_UUID=$WORKER_PLUGIN_RESOURCE_UUID" echo "PRODUCTION_DEPENDENT_RESOURCE_UUIDS=$DEPENDENT_RESOURCE_UUIDS" } >> "$GITHUB_ENV" MIGRATION_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ pre_deployment_command: process.env.MIGRATION_COMMAND, post_deployment_command: process.env.SEED_COMMAND, docker_registry_image_tag: process.env.IMAGE_TAG }))')" curl --fail --silent --show-error --request PATCH \ --header "Authorization: Bearer $COOLIFY_TOKEN" \ --header "Content-Type: application/json" \ --data "$MIGRATION_PAYLOAD" \ "$COOLIFY_API_URL/applications/$WEB_RESOURCE_UUID" >/dev/null IMAGE_TAG_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ docker_registry_image_tag: process.env.IMAGE_TAG }))')" for RESOURCE_UUID in "$API_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do curl --fail --silent --show-error --request PATCH \ --header "Authorization: Bearer $COOLIFY_TOKEN" \ --header "Content-Type: application/json" \ --data "$IMAGE_TAG_PAYLOAD" \ "$COOLIFY_API_URL/applications/$RESOURCE_UUID" >/dev/null done - name: Promote staging images to production aliases run: | if [ "$DEPLOY_ACTION" != "promote" ]; then echo "Promotion skipped for staging" exit 0 fi for TARGET in web api worker; do IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET" docker buildx imagetools create --tag "$IMAGE:main" "$IMAGE:$IMAGE_TAG" done - name: Trigger Coolify production deploy env: COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }} COOLIFY_RESOURCE_UUIDS: ${{ vars.COOLIFY_RESOURCE_UUIDS }} COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }} run: | if [ "$DEPLOY_ACTION" != "promote" ]; then echo "Production deploy skipped for staging" exit 0 fi if [ -z "$COOLIFY_API_URL" ] || [ -z "$PRODUCTION_WEB_RESOURCE_UUID" ] || [ -z "$COOLIFY_TOKEN" ]; then echo "Coolify production configuration is incomplete" exit 1 fi SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN" PRODUCTION_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_WEB_RESOURCE_UUID")" echo "PRODUCTION_WEB_DEPLOYMENT_UUID=$PRODUCTION_WEB_DEPLOYMENT_UUID" >> "$GITHUB_ENV" - name: Wait for Coolify production health env: COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }} COOLIFY_RESOURCE_UUIDS: ${{ vars.COOLIFY_RESOURCE_UUIDS }} COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }} run: | if [ "$DEPLOY_ACTION" != "promote" ]; then echo "Production health wait skipped for staging" exit 0 fi if [ -z "$COOLIFY_API_URL" ] || [ -z "$PRODUCTION_WEB_RESOURCE_UUID" ] || [ -z "$COOLIFY_TOKEN" ]; then echo "Coolify production configuration is incomplete" exit 1 fi if [ -z "$PRODUCTION_WEB_DEPLOYMENT_UUID" ]; then echo "Coolify production deployment UUID is missing" exit 1 fi SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN" node scripts/coolify-deploy.mjs wait "$PRODUCTION_WEB_DEPLOYMENT_UUID" "Coolify production web" verify_live_revision() { APP_JSON="$1" LABEL="$2" REQUIRE_HEALTH_URL="${3:-auto}" EXPECTED_IMAGE_ALIAS="${4:-main}" EXPECTED_SERVICE="${5:-}" HEALTH_URL="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const raw = app.fqdn || app.domain || app.domains || ""; const value = Array.isArray(raw) ? raw[0] : String(raw).split(/[,\s]+/).find(Boolean); process.stdout.write(value ? value.replace(/\/+$/, "") : "");' "$APP_JSON")" HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")" if [ -z "$HEALTH_URL" ]; then if [ "$REQUIRE_HEALTH_URL" = "required" ] || [ "$HTTP_RESOURCE" = "true" ]; then echo "$LABEL is an HTTP resource but has no health URL" exit 1 fi CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")" if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then echo "$LABEL non-HTTP resource image tag does not match expected alias" exit 1 fi echo "$LABEL non-HTTP resource image tag matches expected alias" return fi HEALTH_JSON="$(mktemp)" if ! curl --fail --silent --show-error "$HEALTH_URL/healthz" > "$HEALTH_JSON"; then rm -f "$HEALTH_JSON" echo "$LABEL health endpoint is not ready yet" return 1 fi HEALTH_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$HEALTH_JSON")" HEALTH_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$HEALTH_JSON")" rm -f "$HEALTH_JSON" if [ -n "$EXPECTED_SERVICE" ] && [ "$HEALTH_SERVICE" != "$EXPECTED_SERVICE" ]; then echo "$LABEL health service does not match expected service" exit 1 fi if [ "$HEALTH_REVISION" != "$IMAGE_TAG" ]; then echo "$LABEL health revision does not match promoted image yet" return 1 fi } RESOURCE_INDEX=0 for RESOURCE_UUID in $(printf "%s" "$PRODUCTION_WEB_RESOURCE_UUID" | tr ',' ' '); do RESOURCE_INDEX=$((RESOURCE_INDEX + 1)) HEALTHY=false for ATTEMPT in $(seq 1 24); do APP_JSON="$(mktemp)" curl --fail --silent --show-error \ --header "Authorization: Bearer $COOLIFY_TOKEN" \ "$COOLIFY_API_URL/applications/$RESOURCE_UUID" > "$APP_JSON" STATUS="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.status || "unknown");' "$APP_JSON")" echo "Coolify production resource #$RESOURCE_INDEX status: $STATUS" if [ "$STATUS" = "running:healthy" ]; then if verify_live_revision "$APP_JSON" "Coolify production resource #$RESOURCE_INDEX" required "$IMAGE_TAG" web; then rm -f "$APP_JSON" HEALTHY=true break fi fi rm -f "$APP_JSON" if printf "%s" "$STATUS" | grep -Eiq 'failed|exited|dead'; then echo "Coolify production resource #$RESOURCE_INDEX entered a terminal failure status" exit 1 fi sleep 5 done if [ "$HEALTHY" != "true" ]; then echo "Coolify production resource #$RESOURCE_INDEX did not become healthy in time" exit 1 fi done - name: Prepare production migration status checkout run: | if [ "$DEPLOY_ACTION" != "promote" ]; then echo "Production migration status checkout skipped for staging" exit 0 fi git checkout --detach "$IMAGE_TAG" CHECKOUT_SHA="$(git rev-parse HEAD)" if [ "$CHECKOUT_SHA" != "$IMAGE_TAG" ]; then echo "Production migration status checkout must match the staged image tag" exit 1 fi corepack enable corepack prepare pnpm@10.33.0 --activate pnpm config set store-dir .pnpm-store - name: Cache production migration pnpm store uses: actions/cache@v4 with: path: .pnpm-store key: ${{ runner.os }}-pnpm-promote-${{ hashFiles('pnpm-lock.yaml') }} restore-keys: | ${{ runner.os }}-pnpm- - name: Install production migration dependencies run: | if [ "$DEPLOY_ACTION" != "promote" ]; then echo "Production migration dependency install skipped for staging" exit 0 fi pnpm install --frozen-lockfile --prefer-offline - name: Verify production migration status after web deploy env: DATABASE_URL: ${{ secrets.DATABASE_URL }} # AD-24: Prod-DB liegt im privaten Netz des separaten Prod-Accounts und ist vom CI-Runner # nicht erreichbar. verify-db-migrate-status erkennt P1001 (unreachable) und ueberspringt # den Check sauber (exit 0), wenn dieses Flag gesetzt ist. Die Migration selbst laeuft # ohnehin als Pre-Deployment-Command im web-Container (schlaegt fehl -> Deploy schlaegt # fehl). Greift NUR bei tatsaechlich unerreichbarer DB; ein echter Drift gegen eine # erreichbare DB wuerde weiterhin rot. DB_MIGRATE_STATUS_ALLOW_UNREACHABLE: "true" run: | if [ "$DEPLOY_ACTION" != "promote" ]; then echo "Production migration status check skipped for staging" exit 0 fi if [ -z "$DATABASE_URL" ]; then echo "DATABASE_URL is required for production migration status verification" exit 1 fi pnpm run verify:db-migrate-status - name: Trigger Coolify production dependent deploys env: COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }} COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }} run: | if [ "$DEPLOY_ACTION" != "promote" ]; then echo "Production dependent deploy skipped for staging" exit 0 fi if [ -z "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS" ]; then echo "No production dependent resources to deploy" exit 0 fi if [ -z "$COOLIFY_API_URL" ] || [ -z "$COOLIFY_TOKEN" ]; then echo "Coolify production configuration is incomplete" exit 1 fi SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN" PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS="" for RESOURCE_UUID in $(printf "%s" "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$RESOURCE_UUID")" PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS="${PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS:+$PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS,}$DEPLOYMENT_UUID" done echo "PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS=$PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS" >> "$GITHUB_ENV" - name: Wait for Coolify production dependent health env: COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }} COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }} # AD-24: Prod ist Multi-Server. Worker laufen auf der worker-VM (privates Netz), nur ueber # die control-Bastion erreichbar -> Worker-Health-Verifikation (docker exec) per SSH-ProxyJump. COOLIFY_PROD_SSH_KEY: ${{ secrets.COOLIFY_PROD_SSH_KEY }} COOLIFY_PROD_SSH_KNOWN_HOSTS: ${{ secrets.COOLIFY_PROD_SSH_KNOWN_HOSTS }} COOLIFY_PROD_SSH_HOST: ${{ vars.COOLIFY_PROD_SSH_HOST }} COOLIFY_PROD_WORKER_PRIVATE_IP: ${{ vars.COOLIFY_PROD_WORKER_PRIVATE_IP }} run: | if [ "$DEPLOY_ACTION" != "promote" ]; then echo "Production dependent health wait skipped for staging" exit 0 fi if [ -z "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS" ]; then echo "No production dependent resources to wait for" exit 0 fi if [ -z "$PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS" ]; then echo "Coolify production dependent deployment UUIDs are missing" exit 1 fi if [ -z "$COOLIFY_API_URL" ] || [ -z "$COOLIFY_TOKEN" ]; then echo "Coolify production configuration is incomplete" exit 1 fi SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN" DEPLOYMENT_INDEX=0 DEPLOYMENT_WAIT_PIDS="" for DEPLOYMENT_UUID in $(printf "%s" "$PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS" | tr ',' ' '); do DEPLOYMENT_INDEX=$((DEPLOYMENT_INDEX + 1)) node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify production dependent #$DEPLOYMENT_INDEX" & DEPLOYMENT_WAIT_PIDS="$DEPLOYMENT_WAIT_PIDS $!" done DEPLOYMENT_WAIT_FAILED=false for PID in $DEPLOYMENT_WAIT_PIDS; do if ! wait "$PID"; then DEPLOYMENT_WAIT_FAILED=true fi done if [ "$DEPLOYMENT_WAIT_FAILED" = "true" ]; then echo "One or more Coolify production dependent deployment waits failed" exit 1 fi # SSH helper for worker health verification (Coolify has no exec REST API). AD-24: Prod ist # Multi-Server -> Worker laufen auf der worker-VM (privates Netz), erreichbar nur ueber die # control-Bastion. ProxyJump + gepinnte Host-Keys (kein StrictHostKeyChecking=no). Secrets # werden via Zwischen-Variable in Dateien geschrieben (nie stdout/Logs). if [ -z "$COOLIFY_PROD_SSH_HOST" ] || [ -z "$COOLIFY_PROD_SSH_KEY" ] || [ -z "$COOLIFY_PROD_SSH_KNOWN_HOSTS" ] || [ -z "$COOLIFY_PROD_WORKER_PRIVATE_IP" ]; then echo "Coolify production SSH configuration is incomplete (worker health verification requires SSH + docker exec)" exit 1 fi REMOTE_SSH_KEY_FILE="$(mktemp)" REMOTE_SSH_KNOWN_HOSTS="$(mktemp)" trap 'rm -f "$REMOTE_SSH_KEY_FILE" "$REMOTE_SSH_KNOWN_HOSTS"' EXIT REMOTE_SSH_KEY_DATA="$COOLIFY_PROD_SSH_KEY" printf '%s\n' "$REMOTE_SSH_KEY_DATA" > "$REMOTE_SSH_KEY_FILE" unset REMOTE_SSH_KEY_DATA chmod 600 "$REMOTE_SSH_KEY_FILE" REMOTE_SSH_KNOWN_HOSTS_DATA="$COOLIFY_PROD_SSH_KNOWN_HOSTS" printf '%s\n' "$REMOTE_SSH_KNOWN_HOSTS_DATA" > "$REMOTE_SSH_KNOWN_HOSTS" unset REMOTE_SSH_KNOWN_HOSTS_DATA # ProxyCommand (nicht ProxyJump): ProxyJump reicht den -i-Key nicht zuverlaessig an den # Jump-Hop weiter (MaxAuthTries -> Permission denied). ProxyCommand gibt der Bastion den Key # explizit (IdentitiesOnly), Host-Keys fuer beide Hops gepinnt. REMOTE_SSH_PROXY="ssh -i $REMOTE_SSH_KEY_FILE -o IdentitiesOnly=yes -o BatchMode=yes -o ConnectTimeout=20 -o UserKnownHostsFile=$REMOTE_SSH_KNOWN_HOSTS -o StrictHostKeyChecking=yes -W %h:%p root@${COOLIFY_PROD_SSH_HOST}" remote_ssh() { ssh -i "$REMOTE_SSH_KEY_FILE" -o IdentitiesOnly=yes -o BatchMode=yes -o ConnectTimeout=20 -o UserKnownHostsFile="$REMOTE_SSH_KNOWN_HOSTS" -o StrictHostKeyChecking=yes -o ProxyCommand="$REMOTE_SSH_PROXY" "root@${COOLIFY_PROD_WORKER_PRIVATE_IP}" "$@" } verify_live_revision() { APP_JSON="$1" LABEL="$2" REQUIRE_HEALTH_URL="${3:-auto}" EXPECTED_IMAGE_ALIAS="${4:-main}" EXPECTED_SERVICE="${5:-}" RESOURCE_KIND="${6:-auto}" EXPECTED_WORKER_GROUP="${7:-}" HEALTH_URL="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const raw = app.fqdn || app.domain || app.domains || ""; const value = Array.isArray(raw) ? raw[0] : String(raw).split(/[,\s]+/).find(Boolean); process.stdout.write(value ? value.replace(/\/+$/, "") : "");' "$APP_JSON")" HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")" WORKER_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])worker(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")" if [ "$RESOURCE_KIND" = "worker" ] || [ "$WORKER_RESOURCE" = "true" ]; then CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")" if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then echo "$LABEL worker image tag does not match expected alias" exit 1 fi # Coolify 4.x exposes no command-execution REST API, so verify worker # health by running the one-shot health probe inside the live container # over SSH (docker exec) on the Coolify host. Fail-closed: any SSH / # container-resolution / exec failure blocks the deploy. WORKER_CONTAINER="$(remote_ssh "docker ps --filter name=$RESOURCE_UUID --filter status=running --format '{{.Names}}' | head -n1" 2>/dev/null | tr -d '\r')" if [ -z "$WORKER_CONTAINER" ]; then echo "$LABEL worker container not found on Coolify host" exit 1 fi # health.js exits non-zero when the worker reports a non-ready # status (e.g. "misconfigured"), yet it still prints the diagnostic # JSON (status + errors) to stdout. Capture stdout regardless of the # exit code (|| true) so the real reason surfaces in the checks below # instead of a generic "command unavailable". Empty output means the # docker exec itself could not run (container gone, SSH failure, ...). WORKER_HEALTH_OUTPUT="$(remote_ssh "docker exec $WORKER_CONTAINER node apps/worker/dist/health.js" 2>/dev/null | tr -d '\r' || true)" if [ -z "$WORKER_HEALTH_OUTPUT" ]; then echo "$LABEL worker docker exec health command unavailable (no output from health probe)" exit 1 fi WORKER_HEALTH_JSON="$(mktemp)" printf "%s" "$WORKER_HEALTH_OUTPUT" > "$WORKER_HEALTH_JSON" if ! WORKER_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$WORKER_HEALTH_JSON" 2>/dev/null)"; then rm -f "$WORKER_HEALTH_JSON" echo "$LABEL worker execute output is not JSON" exit 1 fi WORKER_STATUS="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.status || "");' "$WORKER_HEALTH_JSON")" WORKER_ENABLED="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.workerEnabled === true ? "true" : "false");' "$WORKER_HEALTH_JSON")" WORKER_GROUP="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.workerGroup || "");' "$WORKER_HEALTH_JSON")" WORKER_QUEUES="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(Array.isArray(payload.queues) ? payload.queues.join(",") : "");' "$WORKER_HEALTH_JSON")" WORKER_HEALTH_ERRORS="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(Array.isArray(payload.errors) ? payload.errors.join("; ") : "");' "$WORKER_HEALTH_JSON")" rm -f "$WORKER_HEALTH_JSON" if [ "$WORKER_REVISION" != "$IMAGE_TAG" ]; then echo "$LABEL worker health revision does not match image tag yet" return 1 fi if [ "$WORKER_ENABLED" != "true" ]; then echo "$LABEL worker health reports workerEnabled=false" exit 1 fi if [ "$WORKER_STATUS" != "ready" ]; then echo "$LABEL worker health status is not ready: '$WORKER_STATUS'${WORKER_HEALTH_ERRORS:+ (errors: $WORKER_HEALTH_ERRORS)}" exit 1 fi if [ -n "$EXPECTED_WORKER_GROUP" ] && [ "$WORKER_GROUP" != "$EXPECTED_WORKER_GROUP" ]; then echo "$LABEL worker group does not match expected queue group" exit 1 fi EXPECTED_WORKER_QUEUES="" case "$EXPECTED_WORKER_GROUP" in general) EXPECTED_WORKER_QUEUES="toolctrl.notifications toolctrl.maintenance toolctrl.file-scan" ;; mail) EXPECTED_WORKER_QUEUES="toolctrl.email" ;; import) if [ -n "$WORKER_QUEUES" ]; then EXPECTED_WORKER_QUEUES="toolctrl.imports toolctrl.exports" fi ;; webhook) EXPECTED_WORKER_QUEUES="toolctrl.ai-chatbot toolctrl.browser-automation" ;; plugin) EXPECTED_WORKER_QUEUES="toolctrl.plugin-invoke" ;; esac for EXPECTED_WORKER_QUEUE in $EXPECTED_WORKER_QUEUES; do if ! printf ",%s," "$WORKER_QUEUES" | grep -Fq ",$EXPECTED_WORKER_QUEUE,"; then echo "$LABEL worker queues do not match expected queue group" exit 1 fi done echo "$LABEL worker live revision matches image tag" return fi if [ -z "$HEALTH_URL" ]; then if [ "$REQUIRE_HEALTH_URL" = "required" ] || [ "$HTTP_RESOURCE" = "true" ]; then echo "$LABEL is an HTTP resource but has no health URL" exit 1 fi CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")" if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then echo "$LABEL non-HTTP resource image tag does not match expected alias" exit 1 fi echo "$LABEL non-HTTP resource image tag matches expected alias" return fi HEALTH_JSON="$(mktemp)" if ! curl --fail --silent --show-error "$HEALTH_URL/healthz" > "$HEALTH_JSON"; then rm -f "$HEALTH_JSON" echo "$LABEL health endpoint is not ready yet" return 1 fi HEALTH_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$HEALTH_JSON")" HEALTH_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$HEALTH_JSON")" rm -f "$HEALTH_JSON" if [ -n "$EXPECTED_SERVICE" ] && [ "$HEALTH_SERVICE" != "$EXPECTED_SERVICE" ]; then echo "$LABEL health service does not match expected service" exit 1 fi if [ "$HEALTH_REVISION" != "$IMAGE_TAG" ]; then echo "$LABEL health revision does not match promoted image yet" return 1 fi } check_dependent_resource() { RESOURCE_UUID="$1" RESOURCE_INDEX="$2" HEALTHY=false for ATTEMPT in $(seq 1 24); do APP_JSON="$(mktemp)" curl --fail --silent --show-error \ --header "Authorization: Bearer $COOLIFY_TOKEN" \ "$COOLIFY_API_URL/applications/$RESOURCE_UUID" > "$APP_JSON" STATUS="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.status || "unknown");' "$APP_JSON")" echo "Coolify production dependent resource #$RESOURCE_INDEX status: $STATUS" if [ "$STATUS" = "running:healthy" ]; then RESOURCE_KIND="api" EXPECTED_SERVICE="api" EXPECTED_WORKER_GROUP="" if printf ",%s," "$PRODUCTION_WORKER_RESOURCE_UUIDS" | grep -Fq ",$RESOURCE_UUID,"; then RESOURCE_KIND="worker" EXPECTED_SERVICE="" case "$RESOURCE_UUID" in "$PRODUCTION_WORKER_GENERAL_RESOURCE_UUID") EXPECTED_WORKER_GROUP="general" ;; "$PRODUCTION_WORKER_MAIL_RESOURCE_UUID") EXPECTED_WORKER_GROUP="mail" ;; "$PRODUCTION_WORKER_IMPORT_RESOURCE_UUID") EXPECTED_WORKER_GROUP="import" ;; "$PRODUCTION_WORKER_WEBHOOK_RESOURCE_UUID") EXPECTED_WORKER_GROUP="webhook" ;; "$PRODUCTION_WORKER_PLUGIN_RESOURCE_UUID") EXPECTED_WORKER_GROUP="plugin" ;; esac fi if verify_live_revision "$APP_JSON" "Coolify production dependent resource #$RESOURCE_INDEX" auto "$IMAGE_TAG" "$EXPECTED_SERVICE" "$RESOURCE_KIND" "$EXPECTED_WORKER_GROUP"; then rm -f "$APP_JSON" HEALTHY=true break fi fi rm -f "$APP_JSON" if printf "%s" "$STATUS" | grep -Eiq 'failed|exited|dead'; then echo "Coolify production dependent resource #$RESOURCE_INDEX entered a terminal failure status" exit 1 fi sleep 5 done if [ "$HEALTHY" != "true" ]; then echo "Coolify production dependent resource #$RESOURCE_INDEX did not become healthy in time" exit 1 fi } RESOURCE_INDEX=0 RESOURCE_WAIT_PIDS="" for RESOURCE_UUID in $(printf "%s" "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do RESOURCE_INDEX=$((RESOURCE_INDEX + 1)) check_dependent_resource "$RESOURCE_UUID" "$RESOURCE_INDEX" & RESOURCE_WAIT_PIDS="$RESOURCE_WAIT_PIDS $!" done RESOURCE_WAIT_FAILED=false for PID in $RESOURCE_WAIT_PIDS; do if ! wait "$PID"; then RESOURCE_WAIT_FAILED=true fi done if [ "$RESOURCE_WAIT_FAILED" = "true" ]; then echo "One or more Coolify production dependent resources failed health checks" exit 1 fi - name: Mark Sentry deploy if: success() env: SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} SENTRY_ORG: ${{ secrets.SENTRY_ORG }} SENTRY_URL: ${{ secrets.SENTRY_URL }} run: | if [ -z "${SENTRY_AUTH_TOKEN:-}" ] || [ -z "${SENTRY_ORG:-}" ] || [ -z "${SENTRY_URL:-}" ]; then echo "Sentry secrets not configured; skipping deploy marker" exit 0 fi if [ "$DEPLOY_ACTION" = "promote" ]; then SENTRY_ENVIRONMENT=production RELEASE_SHA="${IMAGE_TAG}" else SENTRY_ENVIRONMENT=staging RELEASE_SHA="${SHA}" fi for PROJECT in boilerplate-api boilerplate-worker boilerplate-web; do bash scripts/ci/sentry-release.sh \ --project "$PROJECT" \ --environment "$SENTRY_ENVIRONMENT" \ --release "$RELEASE_SHA" \ --sourcemaps none done - name: Purge asset CDN # Whole-zone purge against Bunny.net so the edge picks up the new # hash-named bundles immediately. The helper script no-ops when # BUNNY_API_KEY is empty, so this step is safe to wire even before # the operator has provisioned a Bunny pull zone. # See docs/ops/asset-cdn.md for the one-time setup. if: success() env: BUNNY_PULL_ZONE_ID: ${{ secrets.BUNNY_PULL_ZONE_ID }} BUNNY_API_KEY: ${{ secrets.BUNNY_API_KEY }} ASSET_CDN_URL: ${{ vars.ASSET_CDN_URL }} run: | if [ -z "${BUNNY_API_KEY:-}" ] || [ -z "${BUNNY_PULL_ZONE_ID:-}" ]; then echo "BUNNY_API_KEY / BUNNY_PULL_ZONE_ID not configured; skipping CDN purge" exit 0 fi pnpm exec tsx scripts/cdn/bunny-purge.ts