Files
HiveGrid-Wireframes/03 Quellen und Referenzen/hivegrid-cloud/.gitea/workflows/deploy.yml
T
raffaelandClaude Opus 5.5 d09f79b673 chore(infra): Gitea-Host auf git.hg-int.dev umstellen
Gitea läuft seit 2026-11-11 auf hivegrid-control (git.hg-int.dev);
git.tool-ctrl.de wird abgeschaltet.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 18:53:32 +01:00

1904 lines
113 KiB
YAML

name: Stage main / Promote staging to production
on:
push:
branches: [main]
# Skip the staging deploy for test-/docs-only merges (e.g. coverage waves, doc updates).
# paths-ignore skips the run only when EVERY changed file matches a pattern; any app or
# config source change still deploys. Manual deploys via workflow_dispatch are unaffected.
paths-ignore:
- "**/*.test.ts"
- "**/*.test.tsx"
- "**/*.test.mjs"
- "**/*.test.js"
- "**/*.spec.ts"
- "docs/**"
- ".planning/**"
- "**/*.md"
- ".gitea/workflows/**"
# Exact CI-only Docker smoke tooling. These files are never copied into
# a runtime image and cannot change a deployed artifact.
- "scripts/ci/docker-smoke-suite.sh"
- "scripts/docker-api-runtime-smoke.sh"
- "scripts/docker-hocuspocus-runtime-smoke.sh"
- "scripts/docker-mcp-runtime-smoke.sh"
- "scripts/docker-portal-domain-controller-runtime-smoke.sh"
- "scripts/docker-web-runtime-smoke.sh"
- "scripts/docker-worker-runtime-smoke.sh"
- "scripts/lib/docker-rls-role-fixture.sh"
- "scripts/lib/docker-runtime-smoke.sh"
- "scripts/lib/run-command-with-timeout.mjs"
- "scripts/verify-gitea-ci-workflow.mjs"
- "scripts/verify-portal-domain-controller-deploy.mjs"
# A manual run is intentionally production-only and input-free. Starting it
# is the operator's audited confirmation that backup/restore-point readiness
# was checked; the workflow discovers the release that is live on staging.
workflow_dispatch:
env:
HIVEGRID_GITEA_HOST_IP: ${{ vars.HIVEGRID_GITEA_HOST_IP }}
concurrency:
group: hivegrid-deploy
cancel-in-progress: false
jobs:
stage-or-promote:
name: Stage or promote Docker images
# The dedicated deploy runner retains one executor from static preflight through
# rollout, avoiding a second queue acquisition between two dependent jobs.
runs-on: toolctrl-deploy
timeout-minutes: 90
env:
REGISTRY: git.hg-int.dev
IMAGE_NAMESPACE: toolctrl-labs
IMAGE_PREFIX: hivegrid
DEPLOY_BUILDER_NAME: hivegrid-deploy-v0312-gc1
DEPLOY_BUILDKIT_IMAGE: moby/buildkit:v0.31.2@sha256:2f5adac4ecd194d9f8c10b7b5d7bceb5186853db1b26e5abd3a657af0b7e26ec
SHA: ${{ gitea.sha }}
steps:
- name: Resolve deployment action
run: |
REF="${GITEA_REF:-${{ gitea.ref }}}"
EVENT_NAME="${GITEA_EVENT_NAME:-${{ gitea.event_name }}}"
if [ "$EVENT_NAME" = "push" ] && [ "$REF" = "refs/heads/main" ]; then
DEPLOY_ACTION=stage
elif [ "$EVENT_NAME" = "workflow_dispatch" ]; then
DEPLOY_ACTION=promote
if [ "$REF" != "refs/heads/main" ]; then
echo "Promote must be dispatched from refs/heads/main"
exit 1
fi
else
echo "Unsupported deployment action: event=$EVENT_NAME ref=$REF"
exit 1
fi
if ! printf '%s' "$SHA" | grep -Eq '^[0-9a-f]{40}$'; then
echo "Gitea SHA must be a full 40-character commit SHA"
exit 1
fi
if [ "$DEPLOY_ACTION" = "stage" ]; then
IMAGE_TAG="$SHA"
echo "IMAGE_TAG=$IMAGE_TAG" >> "$GITHUB_ENV"
fi
echo "DEPLOY_ACTION=$DEPLOY_ACTION" >> "$GITHUB_ENV"
- name: Setup hosts override and Docker CLI
run: |
if [ -n "${HIVEGRID_GITEA_HOST_IP:-}" ]; then
if ! HIVEGRID_GITEA_HOST_IP="$HIVEGRID_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.HIVEGRID_GITEA_HOST_IP || "") ? 0 : 1)'; then
echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token"
exit 1
fi
printf '%s git.hg-int.dev\n' "$HIVEGRID_GITEA_HOST_IP" >> /etc/hosts
fi
MISSING_TOOLS=""
for TOOL in curl git; do
if ! command -v "$TOOL" >/dev/null 2>&1; then
MISSING_TOOLS="$MISSING_TOOLS $TOOL"
fi
done
if ! command -v ssh >/dev/null 2>&1; then
MISSING_TOOLS="$MISSING_TOOLS openssh-client"
fi
if [ -n "$MISSING_TOOLS" ]; then
if ! command -v apt-get >/dev/null 2>&1 || [ "$(id -u)" != "0" ]; then
echo "Missing required tools and apt-get is not available as root:$MISSING_TOOLS"
exit 1
fi
apt-get update -qq
apt-get install -qq -y --no-install-recommends ca-certificates $MISSING_TOOLS
fi
if ! command -v docker >/dev/null 2>&1 || ! docker buildx version >/dev/null 2>&1; then
if ! command -v apt-get >/dev/null 2>&1 || [ "$(id -u)" != "0" ]; then
echo "Docker CLI/buildx missing and apt-get is not available as root"
exit 1
fi
install -m 0755 -d /etc/apt/keyrings
curl -fsSL --connect-timeout 10 --max-time 30 https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
chmod a+r /etc/apt/keyrings/docker.asc
. /etc/os-release
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian ${VERSION_CODENAME} stable" > /etc/apt/sources.list.d/docker.list
apt-get update -qq
apt-get install -qq -y --no-install-recommends docker-ce-cli docker-buildx-plugin
fi
docker --version
docker buildx version
- name: Checkout code
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
with:
fetch-depth: 0
- name: Quiesce orphaned deploy BuildKit
run: |
BUILDER_CONTAINER="buildx_buildkit_${DEPLOY_BUILDER_NAME}0"
if docker inspect "$BUILDER_CONTAINER" >/dev/null 2>&1 && [ "$(docker inspect --format '{{.State.Running}}' "$BUILDER_CONTAINER")" = "true" ]; then
docker stop "$BUILDER_CONTAINER"
fi
- name: Set up Node.js
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
with:
node-version-file: .nvmrc
- name: Persist Node.js toolchain
run: bash scripts/ci/persist-node-toolchain.sh
# Keep the former preflight fail-closed inside the retained deploy job. It
# verifies the workflow revision selected by Gitea before a promotion detaches
# the worktree to its older staged application revision.
- name: Verify deploy wiring
run: |
node scripts/verify-gitea-deploy-workflow.mjs
node scripts/verify-deploy-readiness.mjs
node scripts/verify-docker-baseline.mjs
node scripts/verify-docker-non-root.mjs
node scripts/verify-portal-domain-controller-deploy.mjs
node --check scripts/coolify-deploy.mjs
node --check scripts/coolify-portal-domain-controller.mjs
node --check scripts/coolify-ensure-worker-groups.mjs
node --check scripts/ci/verify-deploy-buildkit-config.mjs
bash -n scripts/coolify-prepull-images.sh
bash -n scripts/ci/install-cosign.sh
- name: Resolve live staging release
run: |
if [ "$DEPLOY_ACTION" != "promote" ]; then
echo "Live staging release resolution skipped for staging"
exit 0
fi
STAGING_HEALTH_JSON="$(mktemp)"
trap 'rm -f "$STAGING_HEALTH_JSON"' EXIT
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 \
https://staging-app.hivegrid.de/healthz > "$STAGING_HEALTH_JSON"
STAGED_SHA="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$STAGING_HEALTH_JSON")"
STAGING_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$STAGING_HEALTH_JSON")"
STAGING_SURFACE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.surface || "");' "$STAGING_HEALTH_JSON")"
if [ "$STAGING_SERVICE" != "hivegrid-web" ] || [ "$STAGING_SURFACE" != "core" ]; then
echo "Staging health endpoint returned the wrong service or surface"
exit 1
fi
if ! printf '%s' "$STAGED_SHA" | grep -Eq '^[0-9a-f]{40}$'; then
echo "Staging health endpoint must return a full 40-character commit SHA"
exit 1
fi
git fetch --no-tags origin main
git cat-file -e "$STAGED_SHA^{commit}"
git merge-base --is-ancestor "$STAGED_SHA" origin/main
echo "IMAGE_TAG=$STAGED_SHA" >> "$GITHUB_ENV"
echo "Resolved live staging release $STAGED_SHA for production promotion"
- name: Verify checkout revision
run: |
git fetch --no-tags origin main
git cat-file -e "$IMAGE_TAG^{commit}"
git merge-base --is-ancestor "$IMAGE_TAG" origin/main
git checkout --detach "$IMAGE_TAG"
CHECKOUT_SHA="$(git rev-parse HEAD)"
if [ "$CHECKOUT_SHA" != "$IMAGE_TAG" ]; then
echo "Checked out revision must match the exact deployment image tag"
exit 1
fi
if [ "$DEPLOY_ACTION" = "stage" ] && [ "$IMAGE_TAG" != "$SHA" ]; then
echo "Staging deployment image tag must match the Gitea SHA"
exit 1
fi
corepack enable --install-directory /usr/local/bin
corepack prepare pnpm@11.12.0 --activate
pnpm config set store-dir .pnpm-store
- name: Cache exact-revision pnpm store
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
with:
path: .pnpm-store
key: ${{ runner.os }}-pnpm-deploy-${{ hashFiles('pnpm-lock.yaml') }}
restore-keys: |
${{ runner.os }}-pnpm-
- name: Install exact-revision dependencies
run: pnpm install --frozen-lockfile --prefer-offline
- name: Verify exact-revision tenant policy contract
run: pnpm run verify:tenant-policy-contract
- name: Verify exact-revision RLS release readiness
run: pnpm run verify:rls-release-readiness
- name: Verify exact-revision live RLS isolation
run: pnpm run verify:rls-live
env:
HIVEGRID_RLS_REQUIRE_MIGRATION_CHAIN_PARITY: "1"
- name: Login to Gitea Registry
run: echo "${{ secrets.REGISTRY_PAT }}" | docker login "$REGISTRY" --username "${{ secrets.REGISTRY_USER }}" --password-stdin
- name: Verify staged promotion provenance
run: |
if [ "$DEPLOY_ACTION" != "promote" ]; then
echo "Staged promotion provenance skipped for staging"
exit 0
fi
git fetch --no-tags origin main
git cat-file -e "$IMAGE_TAG^{commit}"
git merge-base --is-ancestor "$IMAGE_TAG" origin/main
for TARGET in web api worker mcp hocuspocus portal-domain-controller; do
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
SHA_DIGEST="$(docker buildx imagetools inspect "$IMAGE:$IMAGE_TAG" --format '{{json .Manifest.Digest}}')"
STAGING_DIGEST="$(docker buildx imagetools inspect "$IMAGE:staging" --format '{{json .Manifest.Digest}}')"
if [ -z "$SHA_DIGEST" ] || [ "$SHA_DIGEST" != "$STAGING_DIGEST" ]; then
echo "Staged image digest for $TARGET does not match $IMAGE_TAG"
exit 1
fi
done
# Docker Engine 29.2.1 embeds BuildKit 0.27.1, whose build-history
# listener can panic the whole daemon after a history deletion. Keep
# deploy builds on one digest-pinned docker-container builder until the
# homeserver Engine is upgraded to the fixed 29.4+ line. Its stable name
# preserves bounded package-cache mounts between staging runs.
- name: Start isolated deploy BuildKit
if: env.DEPLOY_ACTION == 'stage'
run: |
set -euo pipefail
BUILDER_CONTAINER="buildx_buildkit_${DEPLOY_BUILDER_NAME}0"
if docker inspect "$BUILDER_CONTAINER" >/dev/null 2>&1; then
ACTUAL_BUILDKIT_IMAGE="$(docker inspect --format '{{.Config.Image}}' "$BUILDER_CONTAINER")"
if [ "$ACTUAL_BUILDKIT_IMAGE" != "$DEPLOY_BUILDKIT_IMAGE" ]; then
echo "Existing deploy builder image does not match the pinned BuildKit image"
exit 1
fi
ACTUAL_RESTART_POLICY="$(docker inspect --format '{{.HostConfig.RestartPolicy.Name}}' "$BUILDER_CONTAINER")"
if [ "$ACTUAL_RESTART_POLICY" != "no" ]; then
echo "Existing deploy builder must use restart-policy=no"
exit 1
fi
fi
if docker buildx inspect "$DEPLOY_BUILDER_NAME" >/dev/null 2>&1; then
docker buildx inspect --bootstrap "$DEPLOY_BUILDER_NAME"
else
docker buildx create \
--name "$DEPLOY_BUILDER_NAME" \
--driver docker-container \
--driver-opt "image=$DEPLOY_BUILDKIT_IMAGE" \
--driver-opt "restart-policy=no" \
--buildkitd-config scripts/ci/deploy-buildkitd.toml \
--bootstrap
fi
ACTUAL_BUILDKIT_IMAGE="$(docker inspect --format '{{.Config.Image}}' "$BUILDER_CONTAINER")"
ACTUAL_RESTART_POLICY="$(docker inspect --format '{{.HostConfig.RestartPolicy.Name}}' "$BUILDER_CONTAINER")"
if [ "$ACTUAL_BUILDKIT_IMAGE" != "$DEPLOY_BUILDKIT_IMAGE" ] || [ "$ACTUAL_RESTART_POLICY" != "no" ]; then
echo "Running deploy builder does not match the pinned image and restart policy"
exit 1
fi
docker exec "$BUILDER_CONTAINER" cat /etc/buildkit/buildkitd.toml \
| node scripts/ci/verify-deploy-buildkit-config.mjs scripts/ci/deploy-buildkitd.toml
if [ -z "${HIVEGRID_GITEA_HOST_IP:-}" ]; then
echo "HIVEGRID_GITEA_HOST_IP is required for the isolated deploy builder"
exit 1
fi
if ! HIVEGRID_GITEA_HOST_IP="$HIVEGRID_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.HIVEGRID_GITEA_HOST_IP || "") ? 0 : 1)'; then
echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token"
exit 1
fi
docker exec \
--env HIVEGRID_GITEA_HOST_IP="$HIVEGRID_GITEA_HOST_IP" \
--interactive \
"$BUILDER_CONTAINER" \
sh -eu <<'HIVEGRID_BUILDKIT_HOST_OVERRIDE'
HOSTS_TMP="$(mktemp)"
trap 'rm -f "$HOSTS_TMP"' EXIT HUP INT TERM
awk -v host="git.hg-int.dev" '
{
rewritten = ""
for (field = 1; field <= NF; field += 1) {
if (field > 1 && $field == host) {
continue
}
rewritten = rewritten == "" ? $field : rewritten " " $field
}
if (rewritten != "") {
print rewritten
}
}
' /etc/hosts > "$HOSTS_TMP"
cat "$HOSTS_TMP" > /etc/hosts
printf '%s git.hg-int.dev\n' "$HIVEGRID_GITEA_HOST_IP" >> /etc/hosts
BUILDKIT_HOST_COUNT="$(awk -v host="git.hg-int.dev" '{ for (field = 2; field <= NF; field += 1) if ($field == host) count += 1 } END { print count + 0 }' /etc/hosts)"
BUILDKIT_EXACT_HOST_COUNT="$(awk -v host="git.hg-int.dev" -v ip="$HIVEGRID_GITEA_HOST_IP" '$1 == ip { for (field = 2; field <= NF; field += 1) if ($field == host) count += 1 } END { print count + 0 }' /etc/hosts)"
if [ "$BUILDKIT_HOST_COUNT" -ne 1 ] || [ "$BUILDKIT_EXACT_HOST_COUNT" -ne 1 ]; then
echo "BuildKit registry host override did not converge to exactly one validated mapping"
exit 1
fi
HIVEGRID_BUILDKIT_HOST_OVERRIDE
docker buildx inspect "$DEPLOY_BUILDER_NAME"
- name: Build and push staging images
env:
VITE_SENTRY_DSN_WEB: ${{ secrets.VITE_SENTRY_DSN_WEB }}
VITE_SENTRY_ENVIRONMENT: ${{ secrets.VITE_SENTRY_ENVIRONMENT }}
VITE_OTEL_EXPORTER_OTLP_ENDPOINT: ${{ secrets.VITE_OTEL_EXPORTER_OTLP_ENDPOINT }}
VITE_HOCUSPOCUS_URL: ${{ vars.VITE_HOCUSPOCUS_URL }}
run: |
if [ "$DEPLOY_ACTION" != "stage" ]; then
echo "Stage build skipped for promote"
exit 0
fi
cd "$GITHUB_WORKSPACE"
BUILD_PIDS=()
BUILD_TARGETS=()
for TARGET in web api worker mcp hocuspocus portal-domain-controller; do
(
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
DOCKERFILE="Dockerfile"
if [ "$TARGET" = "hocuspocus" ]; then
DOCKERFILE="apps/hocuspocus/Dockerfile"
fi
WEB_BUILD_ARGS=()
if [ "$TARGET" = "web" ]; then
# VITE_* build args are only meaningful for the web target (Vite
# inlines them into the client bundle). Keeping them web-only
# prevents web env changes from invalidating api/worker caches.
WEB_BUILD_ARGS=(
--build-arg VITE_SENTRY_DSN_WEB="$VITE_SENTRY_DSN_WEB"
--build-arg VITE_SENTRY_ENVIRONMENT="$VITE_SENTRY_ENVIRONMENT"
--build-arg VITE_SENTRY_RELEASE="$SHA"
--build-arg VITE_OTEL_EXPORTER_OTLP_ENDPOINT="$VITE_OTEL_EXPORTER_OTLP_ENDPOINT"
--build-arg VITE_OTEL_SERVICE_NAME="hivegrid-web"
--build-arg VITE_HOCUSPOCUS_URL="$VITE_HOCUSPOCUS_URL"
)
fi
# Import the previous staging image's portable inline cache. The new
# inline cache is embedded in the successful immutable SHA image push
# itself; there is no missing legacy ref lookup or separate registry
# cache export that can fail after the image built.
for ATTEMPT in 1 2 3; do
if docker buildx build --builder "$DEPLOY_BUILDER_NAME" --push --target "$TARGET" --provenance=false --file "$DOCKERFILE" \
--cache-from "type=registry,ref=$IMAGE:staging" \
--cache-to "type=inline" \
--build-arg APP_REVISION="$SHA" \
"${WEB_BUILD_ARGS[@]}" \
--tag "$IMAGE:$IMAGE_TAG" .; then
exit 0
fi
if [ "$ATTEMPT" -lt 3 ]; then
echo "Docker build/push failed for $TARGET on attempt $ATTEMPT; retrying"
sleep $((ATTEMPT * 20))
fi
done
echo "Docker build/push failed for $TARGET after 3 attempts"
exit 1
) &
BUILD_PIDS+=("$!")
BUILD_TARGETS+=("$TARGET")
done
BUILD_FAILED=0
for INDEX in "${!BUILD_PIDS[@]}"; do
if ! wait "${BUILD_PIDS[$INDEX]}"; then
echo "Docker build failed for ${BUILD_TARGETS[$INDEX]}"
BUILD_FAILED=1
fi
done
if [ "$BUILD_FAILED" -ne 0 ]; then
exit "$BUILD_FAILED"
fi
- name: Stop isolated deploy BuildKit
if: always()
run: |
if [ "${DEPLOY_ACTION:-}" != "stage" ]; then
exit 0
fi
BUILDER_CONTAINER="buildx_buildkit_${DEPLOY_BUILDER_NAME}0"
docker inspect "$BUILDER_CONTAINER" >/dev/null
docker stop "$BUILDER_CONTAINER"
- name: Install cosign
if: env.DEPLOY_ACTION == 'stage'
run: bash scripts/ci/install-cosign.sh
- name: Sign published container images
if: env.DEPLOY_ACTION == 'stage'
env:
COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
run: |
if [ -z "${COSIGN_PRIVATE_KEY:-}" ] || [ -z "${COSIGN_PASSWORD:-}" ]; then
echo "Cosign signing skipped: COSIGN_PRIVATE_KEY or COSIGN_PASSWORD not configured"
exit 0
fi
for TARGET in web api worker mcp hocuspocus portal-domain-controller; do
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
cosign sign --yes --key env://COSIGN_PRIVATE_KEY "$IMAGE:$IMAGE_TAG"
done
- name: Validate Coolify staging runtime preflight
env:
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID: ${{ vars.COOLIFY_STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID }}
COOLIFY_STAGING_RESOURCE_UUIDS: ${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }}
COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }}
run: |
if [ "$DEPLOY_ACTION" != "stage" ]; then
echo "Coolify staging inventory validation skipped for promote"
exit 0
fi
RESOLVED_RESOURCE_UUIDS="${STAGING_RESOLVED_RESOURCE_UUIDS:-$COOLIFY_STAGING_RESOURCE_UUIDS}"
WEB_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web"
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
export SELECTED_COOLIFY_API_URL SELECTED_COOLIFY_TOKEN
node scripts/coolify-deploy.mjs assert-runtime-preflight \
"$RESOLVED_RESOURCE_UUIDS" \
"$WEB_IMAGE" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/hocuspocus" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"
node scripts/coolify-portal-domain-controller.mjs assert-runtime \
staging \
"$COOLIFY_STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" \
"$RESOLVED_RESOURCE_UUIDS" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/portal-domain-controller" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"
- name: Run staging one-shot database migration
env:
MIGRATOR_DATABASE_URL: ${{ secrets.STAGING_MIGRATOR_DATABASE_URL }}
run: |
if [ "$DEPLOY_ACTION" != "stage" ]; then
echo "Staging one-shot migration skipped for promote"
exit 0
fi
if [ -z "${MIGRATOR_DATABASE_URL:-}" ]; then
echo "STAGING_MIGRATOR_DATABASE_URL is required for the one-shot migrator"
exit 1
fi
DIRECT_URL="$MIGRATOR_DATABASE_URL" \
docker run --rm --pull=always \
-e DIRECT_URL \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web:$IMAGE_TAG" migrate
- name: Ensure Coolify staging worker groups
env:
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_RESOURCE_UUIDS: ${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }}
COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }}
COOLIFY_STAGING_PROVISION_WORKER_GROUPS: ${{ vars.COOLIFY_STAGING_PROVISION_WORKER_GROUPS }}
COOLIFY_STAGING_PROJECT_UUID: ${{ vars.COOLIFY_STAGING_PROJECT_UUID }}
COOLIFY_STAGING_SERVER_UUID: ${{ vars.COOLIFY_STAGING_SERVER_UUID }}
COOLIFY_STAGING_ENVIRONMENT_NAME: ${{ vars.COOLIFY_STAGING_ENVIRONMENT_NAME }}
COOLIFY_STAGING_ENVIRONMENT_UUID: ${{ vars.COOLIFY_STAGING_ENVIRONMENT_UUID }}
COOLIFY_STAGING_DESTINATION_UUID: ${{ vars.COOLIFY_STAGING_DESTINATION_UUID }}
COOLIFY_STAGING_WORKER_ENV_SOURCE_UUID: ${{ vars.COOLIFY_STAGING_WORKER_ENV_SOURCE_UUID }}
run: |
if [ "$DEPLOY_ACTION" != "stage" ]; then
echo "Coolify staging worker group provisioning skipped for promote"
exit 0
fi
node scripts/coolify-ensure-worker-groups.mjs staging
# The migration has already completed in a short-lived container. This
# step resolves the long-lived resources and rejects any migration/seed
# credential or non-RLS runtime database role before rollout.
- name: Configure Coolify staging runtime resources
env:
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID: ${{ vars.COOLIFY_STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID }}
COOLIFY_STAGING_RESOURCE_UUIDS: ${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }}
COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }}
run: |
if [ "$DEPLOY_ACTION" != "stage" ]; then
echo "Coolify staging runtime configuration skipped for promote"
exit 0
fi
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS="${STAGING_RESOLVED_RESOURCE_UUIDS:-$COOLIFY_STAGING_RESOURCE_UUIDS}"
if [ -z "$SELECTED_COOLIFY_API_URL" ] || [ -z "$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" ] || [ -z "$SELECTED_COOLIFY_TOKEN" ] || [ -z "$COOLIFY_STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" ]; then
echo "Coolify staging configuration is incomplete"
exit 1
fi
WEB_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web"
API_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api"
MCP_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp"
HOCUSPOCUS_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/hocuspocus"
WORKER_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"
PORTAL_DOMAIN_CONTROLLER_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/portal-domain-controller"
PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID="$COOLIFY_STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID"
APPS_JSON="$(mktemp)"
trap 'rm -f "${APPS_JSON:-}"' EXIT
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 \
--header "Authorization: Bearer $SELECTED_COOLIFY_TOKEN" \
"$SELECTED_COOLIFY_API_URL/applications" > "$APPS_JSON"
WEB_SURFACE_RESOURCE_UUIDS="$(SELECTED_COOLIFY_API_URL="$SELECTED_COOLIFY_API_URL" SELECTED_COOLIFY_TOKEN="$SELECTED_COOLIFY_TOKEN" node scripts/coolify-deploy.mjs resolve-web-surfaces "$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" "$WEB_IMAGE")"
WEB_RESOURCE_UUID="${WEB_SURFACE_RESOURCE_UUIDS%%,*}"
REMAINING_WEB_SURFACE_RESOURCE_UUIDS="${WEB_SURFACE_RESOURCE_UUIDS#*,}"
ADMIN_RESOURCE_UUID="${REMAINING_WEB_SURFACE_RESOURCE_UUIDS%%,*}"
PORTAL_RESOURCE_UUID="${REMAINING_WEB_SURFACE_RESOURCE_UUIDS#*,}"
API_RESOURCE_UUID="$(API_IMAGE="$API_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.API_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
MCP_RESOURCE_UUID="$(MCP_IMAGE="$MCP_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.MCP_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
HOCUSPOCUS_RESOURCE_UUID="$(HOCUSPOCUS_IMAGE="$HOCUSPOCUS_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.HOCUSPOCUS_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
select_worker_resource() {
WORKER_QUEUE_GROUP="$1" WORKER_IMAGE="$WORKER_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const group = process.env.WORKER_QUEUE_GROUP || ""; const envPattern = new RegExp(`WORKER_QUEUE_GROUP\\s*(?:=|:)\\s*["'\''"]?${group}\\b`, "i"); const namePattern = new RegExp(`(?:^|[\\s/_-])worker[\\s_-]*${group}(?:$|[\\s:_-])`, "i"); const app = apps.find((candidate) => { const text = [candidate.name, candidate.description, candidate.docker_registry_image_name, candidate.fqdn, candidate.domain, candidate.domains, JSON.stringify(candidate.environment_variables || candidate.envs || candidate.environment || "")].filter(Boolean).join(" "); return candidate.docker_registry_image_name === process.env.WORKER_IMAGE && allowed.has(candidate.uuid) && (envPattern.test(text) || namePattern.test(text)); }); process.stdout.write(app?.uuid || "");' "$APPS_JSON"
}
WORKER_GENERAL_RESOURCE_UUID="$(select_worker_resource general)"
WORKER_MAIL_RESOURCE_UUID="$(select_worker_resource mail)"
WORKER_IMPORT_RESOURCE_UUID="$(select_worker_resource import)"
WORKER_WEBHOOK_RESOURCE_UUID="$(select_worker_resource webhook)"
WORKER_PLUGIN_RESOURCE_UUID="$(select_worker_resource plugin)"
WORKER_RESOURCE_UUIDS="$WORKER_GENERAL_RESOURCE_UUID,$WORKER_MAIL_RESOURCE_UUID,$WORKER_IMPORT_RESOURCE_UUID,$WORKER_WEBHOOK_RESOURCE_UUID,$WORKER_PLUGIN_RESOURCE_UUID"
if [ -z "$WEB_RESOURCE_UUID" ] || [ -z "$ADMIN_RESOURCE_UUID" ] || [ -z "$PORTAL_RESOURCE_UUID" ] || [ -z "$API_RESOURCE_UUID" ] || [ -z "$MCP_RESOURCE_UUID" ] || [ -z "$HOCUSPOCUS_RESOURCE_UUID" ] || [ -z "$WORKER_GENERAL_RESOURCE_UUID" ] || [ -z "$WORKER_MAIL_RESOURCE_UUID" ] || [ -z "$WORKER_IMPORT_RESOURCE_UUID" ] || [ -z "$WORKER_WEBHOOK_RESOURCE_UUID" ] || [ -z "$WORKER_PLUGIN_RESOURCE_UUID" ] || [ -z "$PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" ]; then
echo "Coolify staging hivegrid-web, hivegrid-admin, hivegrid-portal, portal-domain-controller, api, mcp, hocuspocus, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications must all exist"
exit 1
fi
for REQUIRED_RESOURCE_UUID in "$WEB_RESOURCE_UUID" "$ADMIN_RESOURCE_UUID" "$PORTAL_RESOURCE_UUID" "$PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" "$API_RESOURCE_UUID" "$MCP_RESOURCE_UUID" "$HOCUSPOCUS_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
if ! printf ",%s," "$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" | grep -Fq ",$REQUIRED_RESOURCE_UUID,"; then
echo "Coolify staging deploy resources must include hivegrid-web, hivegrid-admin, hivegrid-portal, portal-domain-controller, api, mcp, hocuspocus, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin"
exit 1
fi
done
node scripts/coolify-deploy.mjs assert-no-app-revision-override "$WEB_RESOURCE_UUID"
node scripts/coolify-deploy.mjs assert-no-app-revision-override "$ADMIN_RESOURCE_UUID"
node scripts/coolify-deploy.mjs assert-no-app-revision-override "$PORTAL_RESOURCE_UUID"
node scripts/coolify-deploy.mjs assert-no-app-revision-override "$API_RESOURCE_UUID"
node scripts/coolify-deploy.mjs assert-runtime-db-env "$WEB_RESOURCE_UUID" hivegrid_app web
node scripts/coolify-deploy.mjs assert-runtime-db-env "$ADMIN_RESOURCE_UUID" hivegrid_app admin
node scripts/coolify-deploy.mjs assert-runtime-db-env "$PORTAL_RESOURCE_UUID" hivegrid_portal portal
node scripts/coolify-deploy.mjs assert-runtime-db-env "$API_RESOURCE_UUID" hivegrid_app service
node scripts/coolify-deploy.mjs assert-runtime-db-env "$MCP_RESOURCE_UUID" hivegrid_app service
node scripts/coolify-deploy.mjs assert-runtime-db-env "$HOCUSPOCUS_RESOURCE_UUID" hivegrid_app service
for WORKER_RESOURCE_UUID in "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
node scripts/coolify-deploy.mjs assert-runtime-db-env "$WORKER_RESOURCE_UUID" hivegrid_worker service
done
node scripts/coolify-deploy.mjs assert-web-push-env "$WEB_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID"
DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$MCP_RESOURCE_UUID,$HOCUSPOCUS_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"
WEB_IMAGE_DIGEST="$(docker buildx imagetools inspect "$WEB_IMAGE:$IMAGE_TAG" --format '{{json .Manifest.Digest}}' | tr -d '"')"
if ! printf '%s' "$WEB_IMAGE_DIGEST" | grep -Eq '^sha256:[0-9a-f]{64}$'; then
echo "Staging web image digest is missing or malformed"
exit 1
fi
node scripts/coolify-deploy.mjs set-web-surface-release \
"$WEB_RESOURCE_UUID" \
"$ADMIN_RESOURCE_UUID" \
"$PORTAL_RESOURCE_UUID" \
"$WEB_IMAGE" \
"$IMAGE_TAG" \
"$WEB_IMAGE_DIGEST"
{
echo "STAGING_WEB_RESOURCE_UUID=$WEB_RESOURCE_UUID"
echo "STAGING_ADMIN_RESOURCE_UUID=$ADMIN_RESOURCE_UUID"
echo "STAGING_PORTAL_RESOURCE_UUID=$PORTAL_RESOURCE_UUID"
echo "STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID=$PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID"
echo "STAGING_WEB_IMAGE_DIGEST=$WEB_IMAGE_DIGEST"
echo "STAGING_MCP_RESOURCE_UUID=$MCP_RESOURCE_UUID"
echo "STAGING_HOCUSPOCUS_RESOURCE_UUID=$HOCUSPOCUS_RESOURCE_UUID"
echo "STAGING_WORKER_RESOURCE_UUIDS=$WORKER_RESOURCE_UUIDS"
echo "STAGING_WORKER_GENERAL_RESOURCE_UUID=$WORKER_GENERAL_RESOURCE_UUID"
echo "STAGING_WORKER_MAIL_RESOURCE_UUID=$WORKER_MAIL_RESOURCE_UUID"
echo "STAGING_WORKER_IMPORT_RESOURCE_UUID=$WORKER_IMPORT_RESOURCE_UUID"
echo "STAGING_WORKER_WEBHOOK_RESOURCE_UUID=$WORKER_WEBHOOK_RESOURCE_UUID"
echo "STAGING_WORKER_PLUGIN_RESOURCE_UUID=$WORKER_PLUGIN_RESOURCE_UUID"
echo "STAGING_DEPENDENT_RESOURCE_UUIDS=$DEPENDENT_RESOURCE_UUIDS"
} >> "$GITHUB_ENV"
IMAGE_TAG_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ docker_registry_image_tag: process.env.IMAGE_TAG }))')"
for RESOURCE_UUID in "$API_RESOURCE_UUID" "$MCP_RESOURCE_UUID" "$HOCUSPOCUS_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 --request PATCH \
--header "Authorization: Bearer $SELECTED_COOLIFY_TOKEN" \
--header "Content-Type: application/json" \
--data "$IMAGE_TAG_PAYLOAD" \
"$SELECTED_COOLIFY_API_URL/applications/$RESOURCE_UUID" >/dev/null
done
- name: Publish staging image aliases
run: |
if [ "$DEPLOY_ACTION" != "stage" ]; then
echo "Staging alias publication skipped for promote"
exit 0
fi
for TARGET in web api worker mcp hocuspocus portal-domain-controller; do
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
docker buildx imagetools create --prefer-index=false --tag "$IMAGE:staging" "$IMAGE:$IMAGE_TAG"
done
# Stop the old revision before either web surface changes. BullMQ Worker
# shutdown waits for active jobs; new jobs stay queued until the paired
# controller revision starts after both web surfaces prove this SHA.
- name: Stop and drain Coolify staging portal-domain-controller
env:
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_RESOURCE_UUIDS: ${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }}
COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }}
run: |
if [ "$DEPLOY_ACTION" != "stage" ]; then
echo "Staging portal-domain-controller drain skipped for promote"
exit 0
fi
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS="${STAGING_RESOLVED_RESOURCE_UUIDS:-$COOLIFY_STAGING_RESOURCE_UUIDS}"
STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION="$(node scripts/coolify-portal-domain-controller.mjs snapshot-release \
staging \
"$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" \
"$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/portal-domain-controller" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker")"
echo "STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION=$STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION" >> "$GITHUB_ENV"
echo "STAGING_PORTAL_DOMAIN_CONTROLLER_RECOVERY_ARMED=true" >> "$GITHUB_ENV"
node scripts/coolify-portal-domain-controller.mjs stop-and-drain \
staging \
"$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" \
"$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/portal-domain-controller" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"
- name: Trigger Coolify staging deploy
env:
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_RESOURCE_UUIDS: ${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }}
COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }}
run: |
if [ "$DEPLOY_ACTION" != "stage" ]; then
echo "Staging deploy skipped for promote"
exit 0
fi
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
if [ -z "$SELECTED_COOLIFY_API_URL" ] || [ -z "$STAGING_WEB_RESOURCE_UUID" ] || [ -z "$STAGING_ADMIN_RESOURCE_UUID" ] || [ -z "$STAGING_PORTAL_RESOURCE_UUID" ] || [ -z "$SELECTED_COOLIFY_TOKEN" ]; then
echo "Coolify staging configuration is incomplete"
exit 1
fi
STAGING_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_WEB_RESOURCE_UUID")"
echo "STAGING_WEB_DEPLOYMENT_UUID=$STAGING_WEB_DEPLOYMENT_UUID" >> "$GITHUB_ENV"
node scripts/coolify-deploy.mjs wait \
"$STAGING_WEB_DEPLOYMENT_UUID" \
"Coolify staging hivegrid-web"
STAGING_ADMIN_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_ADMIN_RESOURCE_UUID")"
echo "STAGING_ADMIN_DEPLOYMENT_UUID=$STAGING_ADMIN_DEPLOYMENT_UUID" >> "$GITHUB_ENV"
node scripts/coolify-deploy.mjs wait \
"$STAGING_ADMIN_DEPLOYMENT_UUID" \
"Coolify staging hivegrid-admin"
STAGING_PORTAL_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_PORTAL_RESOURCE_UUID")"
echo "STAGING_PORTAL_DEPLOYMENT_UUID=$STAGING_PORTAL_DEPLOYMENT_UUID" >> "$GITHUB_ENV"
node scripts/coolify-deploy.mjs wait \
"$STAGING_PORTAL_DEPLOYMENT_UUID" \
"Coolify staging hivegrid-portal"
- name: Wait for Coolify staging health
env:
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_RESOURCE_UUIDS: ${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }}
COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }}
run: |
if [ "$DEPLOY_ACTION" != "stage" ]; then
echo "Staging health wait skipped for promote"
exit 0
fi
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
if [ -z "$SELECTED_COOLIFY_API_URL" ] || [ -z "$STAGING_WEB_RESOURCE_UUID" ] || [ -z "$STAGING_ADMIN_RESOURCE_UUID" ] || [ -z "$STAGING_PORTAL_RESOURCE_UUID" ] || [ -z "$STAGING_WEB_IMAGE_DIGEST" ] || [ -z "$SELECTED_COOLIFY_TOKEN" ]; then
echo "Coolify staging configuration is incomplete"
exit 1
fi
WEB_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web"
EXPECTED_WEB_DIGEST_TAG="sha256-${STAGING_WEB_IMAGE_DIGEST#sha256:}"
if [ -z "${STAGING_WEB_DEPLOYMENT_UUID:-}" ] || [ -z "${STAGING_ADMIN_DEPLOYMENT_UUID:-}" ] || [ -z "${STAGING_PORTAL_DEPLOYMENT_UUID:-}" ]; then
echo "Coolify staging web surface deployment UUIDs are missing"
exit 1
fi
verify_live_revision() {
APP_JSON="$1"
LABEL="$2"
REQUIRE_HEALTH_URL="${3:-auto}"
EXPECTED_IMAGE_ALIAS="${4:-staging}"
EXPECTED_SERVICE="${5:-}"
EXPECTED_SURFACE="${6:-}"
HEALTH_URL="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const raw = app.fqdn || app.domain || app.domains || ""; const value = Array.isArray(raw) ? raw[0] : String(raw).split(/[,\s]+/).find(Boolean); process.stdout.write(value ? value.replace(/\/+$/, "") : "");' "$APP_JSON")"
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api|mcp)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
if [ -z "$HEALTH_URL" ]; then
if [ "$REQUIRE_HEALTH_URL" = "required" ] || [ "$HTTP_RESOURCE" = "true" ]; then
echo "$LABEL is an HTTP resource but has no health URL"
exit 1
fi
CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")"
if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then
echo "$LABEL non-HTTP resource image tag does not match expected alias"
exit 1
fi
echo "$LABEL non-HTTP resource image tag matches expected alias"
return
fi
HEALTH_JSON="$(mktemp)"
READINESS_JSON="$(mktemp)"
if ! curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL/healthz" > "$HEALTH_JSON"; then
rm -f "$HEALTH_JSON" "$READINESS_JSON"
echo "$LABEL health endpoint is not ready yet"
return 1
fi
if ! curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL/readyz" > "$READINESS_JSON"; then
rm -f "$HEALTH_JSON" "$READINESS_JSON"
echo "$LABEL readiness endpoint is not ready yet"
return 1
fi
HEALTH_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$HEALTH_JSON")"
HEALTH_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$HEALTH_JSON")"
HEALTH_SURFACE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.surface || "");' "$HEALTH_JSON")"
READINESS_STATUS="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.status || "");' "$READINESS_JSON")"
READINESS_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$READINESS_JSON")"
READINESS_SURFACE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.surface || "");' "$READINESS_JSON")"
READINESS_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$READINESS_JSON")"
rm -f "$HEALTH_JSON" "$READINESS_JSON"
if [ -n "$EXPECTED_SERVICE" ] && [ "$HEALTH_SERVICE" != "$EXPECTED_SERVICE" ]; then
echo "$LABEL health service does not match expected service"
exit 1
fi
if [ -n "$EXPECTED_SURFACE" ] && [ "$HEALTH_SURFACE" != "$EXPECTED_SURFACE" ]; then
echo "$LABEL health surface does not match expected surface"
exit 1
fi
if [ "$HEALTH_REVISION" != "$IMAGE_TAG" ]; then
echo "$LABEL health revision does not match staged image yet"
return 1
fi
if [ "$READINESS_STATUS" != "ready" ]; then
echo "$LABEL readiness status is not ready yet"
return 1
fi
if [ "$READINESS_REVISION" != "$IMAGE_TAG" ]; then
echo "$LABEL readiness revision does not match staged image yet"
return 1
fi
if [ -n "$EXPECTED_SERVICE" ] && [ "$READINESS_SERVICE" != "$EXPECTED_SERVICE" ]; then
echo "$LABEL readiness service does not match expected service"
exit 1
fi
if [ -n "$EXPECTED_SURFACE" ] && [ "$READINESS_SURFACE" != "$EXPECTED_SURFACE" ]; then
echo "$LABEL readiness surface does not match expected surface"
exit 1
fi
}
RESOURCE_INDEX=0
for SURFACE in core admin portal; do
RESOURCE_INDEX=$((RESOURCE_INDEX + 1))
if [ "$SURFACE" = "core" ]; then
RESOURCE_UUID="$STAGING_WEB_RESOURCE_UUID"
EXPECTED_SERVICE="hivegrid-web"
elif [ "$SURFACE" = "admin" ]; then
RESOURCE_UUID="$STAGING_ADMIN_RESOURCE_UUID"
EXPECTED_SERVICE="hivegrid-admin"
else
RESOURCE_UUID="$STAGING_PORTAL_RESOURCE_UUID"
EXPECTED_SERVICE="hivegrid-portal"
fi
HEALTHY=false
for ATTEMPT in $(seq 1 24); do
APP_JSON="$(mktemp)"
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 \
--header "Authorization: Bearer $SELECTED_COOLIFY_TOKEN" \
"$SELECTED_COOLIFY_API_URL/applications/$RESOURCE_UUID" > "$APP_JSON"
STATUS="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.status || "unknown");' "$APP_JSON")"
CONFIGURED_IMAGE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_name || "");' "$APP_JSON")"
CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || "");' "$APP_JSON")"
echo "Coolify staging resource #$RESOURCE_INDEX status: $STATUS"
if [ "$CONFIGURED_IMAGE" != "$WEB_IMAGE" ] || [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_WEB_DIGEST_TAG" ]; then
rm -f "$APP_JSON"
echo "Coolify staging web surface #$RESOURCE_INDEX is not pinned to the paired registry digest"
exit 1
fi
if [ "$STATUS" = "running:healthy" ]; then
if verify_live_revision "$APP_JSON" "Coolify staging resource #$RESOURCE_INDEX" required "$IMAGE_TAG" "$EXPECTED_SERVICE" "$SURFACE"; then
rm -f "$APP_JSON"
HEALTHY=true
break
fi
fi
rm -f "$APP_JSON"
if printf "%s" "$STATUS" | grep -Eiq 'failed|exited|dead'; then
echo "Coolify staging resource #$RESOURCE_INDEX entered a terminal failure status"
exit 1
fi
sleep 5
done
if [ "$HEALTHY" != "true" ]; then
echo "Coolify staging resource #$RESOURCE_INDEX did not become healthy in time"
exit 1
fi
done
ROLLED_OUT_WEB_IMAGE_DIGEST="$(docker buildx imagetools inspect "$WEB_IMAGE:$IMAGE_TAG" --format '{{json .Manifest.Digest}}' | tr -d '"')"
if [ "$ROLLED_OUT_WEB_IMAGE_DIGEST" != "$STAGING_WEB_IMAGE_DIGEST" ]; then
echo "Staging web image digest changed during rollout"
exit 1
fi
node scripts/coolify-deploy.mjs assert-web-surface-release \
"$STAGING_WEB_RESOURCE_UUID" \
"$STAGING_ADMIN_RESOURCE_UUID" \
"$STAGING_PORTAL_RESOURCE_UUID" \
"$WEB_IMAGE" \
"$IMAGE_TAG" \
"$ROLLED_OUT_WEB_IMAGE_DIGEST" \
"$STAGING_WEB_DEPLOYMENT_UUID" \
"$STAGING_ADMIN_DEPLOYMENT_UUID" \
"$STAGING_PORTAL_DEPLOYMENT_UUID"
- name: Trigger Coolify staging dependent deploys
env:
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }}
run: |
if [ "$DEPLOY_ACTION" != "stage" ]; then
echo "Staging dependent deploy skipped for promote"
exit 0
fi
if [ -z "$STAGING_DEPENDENT_RESOURCE_UUIDS" ]; then
echo "No staging dependent resources to deploy"
exit 0
fi
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
if [ -z "$SELECTED_COOLIFY_API_URL" ] || [ -z "$SELECTED_COOLIFY_TOKEN" ]; then
echo "Coolify staging configuration is incomplete"
exit 1
fi
# Keep readiness inside the dependency-free runtime: Coolify runs
# the image-bundled Node command instead of injecting curl/wget.
node scripts/coolify-deploy.mjs configure-mcp-healthcheck "$STAGING_MCP_RESOURCE_UUID" "$IMAGE_TAG"
STAGING_DEPENDENT_DEPLOYMENT_UUIDS=""
STAGING_DEPENDENT_TRIGGER_FAILED=false
DEPLOYMENT_INDEX=0
for RESOURCE_UUID in $(printf "%s" "$STAGING_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do
DEPLOYMENT_INDEX=$((DEPLOYMENT_INDEX + 1))
if DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$RESOURCE_UUID")"; then
STAGING_DEPENDENT_DEPLOYMENT_UUIDS="${STAGING_DEPENDENT_DEPLOYMENT_UUIDS:+$STAGING_DEPENDENT_DEPLOYMENT_UUIDS,}$DEPLOYMENT_UUID"
# Coolify can dispatch the same queued deployment twice when several
# resources are enqueued in one burst. Keep deployments serialized;
# the following health/revision probes still fan out independently.
if ! node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify staging dependent deploy #$DEPLOYMENT_INDEX"; then
echo "A serialized Coolify staging dependent deploy failed"
STAGING_DEPENDENT_TRIGGER_FAILED=true
fi
else
echo "A Coolify staging dependent deploy trigger failed"
STAGING_DEPENDENT_TRIGGER_FAILED=true
fi
done
echo "STAGING_DEPENDENT_DEPLOYMENT_UUIDS=$STAGING_DEPENDENT_DEPLOYMENT_UUIDS" >> "$GITHUB_ENV"
echo "STAGING_DEPENDENT_TRIGGER_FAILED=$STAGING_DEPENDENT_TRIGGER_FAILED" >> "$GITHUB_ENV"
- name: Wait for Coolify staging dependent health
env:
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }}
run: |
if [ "$DEPLOY_ACTION" != "stage" ]; then
echo "Staging dependent health wait skipped for promote"
exit 0
fi
if [ -z "$STAGING_DEPENDENT_RESOURCE_UUIDS" ]; then
echo "No staging dependent resources to wait for"
exit 0
fi
if [ -z "$STAGING_DEPENDENT_DEPLOYMENT_UUIDS" ] && [ "${STAGING_DEPENDENT_TRIGGER_FAILED:-false}" != "true" ]; then
echo "Coolify staging dependent deployment UUIDs are missing"
exit 1
fi
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
if [ -z "$SELECTED_COOLIFY_API_URL" ] || [ -z "$SELECTED_COOLIFY_TOKEN" ]; then
echo "Coolify staging configuration is incomplete"
exit 1
fi
DEPLOYMENT_INDEX=0
DEPLOYMENT_WAIT_PIDS=""
for DEPLOYMENT_UUID in $(printf "%s" "$STAGING_DEPENDENT_DEPLOYMENT_UUIDS" | tr ',' ' '); do
DEPLOYMENT_INDEX=$((DEPLOYMENT_INDEX + 1))
node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify staging dependent #$DEPLOYMENT_INDEX" &
DEPLOYMENT_WAIT_PIDS="$DEPLOYMENT_WAIT_PIDS $!"
done
DEPLOYMENT_WAIT_FAILED=false
for PID in $DEPLOYMENT_WAIT_PIDS; do
if ! wait "$PID"; then
DEPLOYMENT_WAIT_FAILED=true
fi
done
if [ "$DEPLOYMENT_WAIT_FAILED" = "true" ]; then
echo "One or more Coolify staging dependent deployment waits failed"
exit 1
fi
verify_live_revision() {
APP_JSON="$1"
LABEL="$2"
REQUIRE_HEALTH_URL="${3:-auto}"
EXPECTED_IMAGE_ALIAS="${4:-staging}"
EXPECTED_SERVICE="${5:-}"
RESOURCE_KIND="${6:-auto}"
EXPECTED_WORKER_GROUP="${7:-}"
HEALTH_URL="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const raw = app.fqdn || app.domain || app.domains || ""; const value = Array.isArray(raw) ? raw[0] : String(raw).split(/[,\s]+/).find(Boolean); process.stdout.write(value ? value.replace(/\/+$/, "") : "");' "$APP_JSON")"
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api|mcp)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
WORKER_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])worker(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
if [ "$RESOURCE_KIND" = "hocuspocus" ]; then
CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")"
if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then
echo "$LABEL Hocuspocus image tag does not match expected revision"
exit 1
fi
echo "$LABEL Hocuspocus Docker health and image revision are healthy"
return
fi
if [ "$RESOURCE_KIND" = "worker" ] || [ "$WORKER_RESOURCE" = "true" ]; then
CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")"
if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then
echo "$LABEL worker image tag does not match expected alias"
exit 1
fi
if [ -z "$EXPECTED_WORKER_GROUP" ]; then
echo "$LABEL worker group could not be resolved"
exit 1
fi
EXPECTED_HEALTH_COMMAND="node apps/worker/dist/deploy-health.js $IMAGE_TAG $EXPECTED_WORKER_GROUP"
CONFIGURED_HEALTH_COMMAND="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.health_check_command || "");' "$APP_JSON")"
CONFIGURED_HEALTH_TYPE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.health_check_type || "");' "$APP_JSON")"
HEALTH_CHECK_ENABLED="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = app.health_check_enabled; process.stdout.write(value === true || value === "true" || value === 1 || value === "1" ? "true" : "false");' "$APP_JSON")"
CUSTOM_HEALTHCHECK_FOUND="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = app.custom_healthcheck_found; process.stdout.write(value === false || value === "false" || value === 0 || value === "0" ? "false" : value === true || value === "true" || value === 1 || value === "1" ? "true" : "unknown");' "$APP_JSON")"
if [ "$HEALTH_CHECK_ENABLED" != "true" ] || [ "$CONFIGURED_HEALTH_TYPE" != "cmd" ] || [ "$CONFIGURED_HEALTH_COMMAND" != "$EXPECTED_HEALTH_COMMAND" ] || [ "$CUSTOM_HEALTHCHECK_FOUND" != "false" ]; then
echo "$LABEL worker healthcheck is not bound to the deployed revision"
exit 1
fi
echo "$LABEL worker Coolify status and revision-bound command healthcheck are healthy"
return
fi
if [ -z "$HEALTH_URL" ]; then
if [ "$REQUIRE_HEALTH_URL" = "required" ] || [ "$HTTP_RESOURCE" = "true" ]; then
echo "$LABEL is an HTTP resource but has no health URL"
exit 1
fi
CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")"
if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then
echo "$LABEL non-HTTP resource image tag does not match expected alias"
exit 1
fi
echo "$LABEL non-HTTP resource image tag matches expected alias"
return
fi
HEALTH_PATH="/healthz"
if [ "$RESOURCE_KIND" = "mcp" ]; then
HEALTH_PATH="/readyz"
EXPECTED_MCP_HEALTH_COMMAND="node apps/mcp/dist/deploy-health.js $IMAGE_TAG"
CONFIGURED_MCP_HEALTH_COMMAND="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.health_check_command || "");' "$APP_JSON")"
CONFIGURED_MCP_HEALTH_TYPE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.health_check_type || "");' "$APP_JSON")"
MCP_HEALTH_CHECK_ENABLED="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = app.health_check_enabled; process.stdout.write(value === true || value === "true" || value === 1 || value === "1" ? "true" : "false");' "$APP_JSON")"
MCP_CUSTOM_HEALTHCHECK_FOUND="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = app.custom_healthcheck_found; process.stdout.write(value === false || value === "false" || value === 0 || value === "0" ? "false" : value === true || value === "true" || value === 1 || value === "1" ? "true" : "unknown");' "$APP_JSON")"
if [ "$MCP_HEALTH_CHECK_ENABLED" != "true" ] || [ "$CONFIGURED_MCP_HEALTH_TYPE" != "cmd" ] || [ "$CONFIGURED_MCP_HEALTH_COMMAND" != "$EXPECTED_MCP_HEALTH_COMMAND" ] || [ "$MCP_CUSTOM_HEALTHCHECK_FOUND" != "false" ]; then
echo "$LABEL MCP healthcheck is not bound to the deployed revision"
exit 1
fi
fi
HEALTH_JSON="$(mktemp)"
if ! curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL$HEALTH_PATH" > "$HEALTH_JSON"; then
rm -f "$HEALTH_JSON"
echo "$LABEL readiness endpoint is not ready yet"
return 1
fi
HEALTH_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$HEALTH_JSON")"
HEALTH_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$HEALTH_JSON")"
rm -f "$HEALTH_JSON"
if [ -n "$EXPECTED_SERVICE" ] && [ "$HEALTH_SERVICE" != "$EXPECTED_SERVICE" ]; then
echo "$LABEL health service does not match expected service"
exit 1
fi
if [ "$HEALTH_REVISION" != "$IMAGE_TAG" ]; then
echo "$LABEL health revision does not match staged image yet"
return 1
fi
}
check_dependent_resource() {
RESOURCE_UUID="$1"
RESOURCE_INDEX="$2"
HEALTHY=false
for ATTEMPT in $(seq 1 24); do
APP_JSON="$(mktemp)"
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 \
--header "Authorization: Bearer $SELECTED_COOLIFY_TOKEN" \
"$SELECTED_COOLIFY_API_URL/applications/$RESOURCE_UUID" > "$APP_JSON"
STATUS="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.status || "unknown");' "$APP_JSON")"
echo "Coolify staging dependent resource #$RESOURCE_INDEX status: $STATUS"
if [ "$STATUS" = "running:healthy" ]; then
RESOURCE_KIND="api"
EXPECTED_SERVICE="api"
EXPECTED_WORKER_GROUP=""
if [ "$RESOURCE_UUID" = "$STAGING_MCP_RESOURCE_UUID" ]; then
RESOURCE_KIND="mcp"
EXPECTED_SERVICE="mcp"
fi
if [ "$RESOURCE_UUID" = "$STAGING_HOCUSPOCUS_RESOURCE_UUID" ]; then
RESOURCE_KIND="hocuspocus"
EXPECTED_SERVICE=""
fi
if printf ",%s," "$STAGING_WORKER_RESOURCE_UUIDS" | grep -Fq ",$RESOURCE_UUID,"; then
RESOURCE_KIND="worker"
EXPECTED_SERVICE=""
case "$RESOURCE_UUID" in
"$STAGING_WORKER_GENERAL_RESOURCE_UUID") EXPECTED_WORKER_GROUP="general" ;;
"$STAGING_WORKER_MAIL_RESOURCE_UUID") EXPECTED_WORKER_GROUP="mail" ;;
"$STAGING_WORKER_IMPORT_RESOURCE_UUID") EXPECTED_WORKER_GROUP="import" ;;
"$STAGING_WORKER_WEBHOOK_RESOURCE_UUID") EXPECTED_WORKER_GROUP="webhook" ;;
"$STAGING_WORKER_PLUGIN_RESOURCE_UUID") EXPECTED_WORKER_GROUP="plugin" ;;
esac
fi
if verify_live_revision "$APP_JSON" "Coolify staging dependent resource #$RESOURCE_INDEX" auto "$IMAGE_TAG" "$EXPECTED_SERVICE" "$RESOURCE_KIND" "$EXPECTED_WORKER_GROUP"; then
rm -f "$APP_JSON"
HEALTHY=true
break
fi
fi
rm -f "$APP_JSON"
if printf "%s" "$STATUS" | grep -Eiq 'failed|exited|dead'; then
echo "Coolify staging dependent resource #$RESOURCE_INDEX entered a terminal failure status"
exit 1
fi
sleep 5
done
if [ "$HEALTHY" != "true" ]; then
echo "Coolify staging dependent resource #$RESOURCE_INDEX did not become healthy in time"
exit 1
fi
}
RESOURCE_INDEX=0
RESOURCE_WAIT_PIDS=""
for RESOURCE_UUID in $(printf "%s" "$STAGING_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do
RESOURCE_INDEX=$((RESOURCE_INDEX + 1))
check_dependent_resource "$RESOURCE_UUID" "$RESOURCE_INDEX" &
RESOURCE_WAIT_PIDS="$RESOURCE_WAIT_PIDS $!"
done
RESOURCE_WAIT_FAILED=false
for PID in $RESOURCE_WAIT_PIDS; do
if ! wait "$PID"; then
RESOURCE_WAIT_FAILED=true
fi
done
if [ "$RESOURCE_WAIT_FAILED" = "true" ]; then
echo "One or more Coolify staging dependent resources failed health checks"
exit 1
fi
if [ "${STAGING_DEPENDENT_TRIGGER_FAILED:-false}" = "true" ]; then
echo "One or more Coolify staging dependent deploy triggers failed"
exit 1
fi
# Configure and start the paired controller only after both web surfaces
# and worker-general have proved this exact SHA. If rollout fails after
# the drain, the always-on recovery below restores the validated release.
- name: Trigger and verify Coolify staging portal-domain-controller
env:
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_RESOURCE_UUIDS: ${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }}
COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }}
run: |
if [ "$DEPLOY_ACTION" != "stage" ]; then
echo "Staging portal-domain-controller deploy skipped for promote"
exit 0
fi
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS="${STAGING_RESOLVED_RESOURCE_UUIDS:-$COOLIFY_STAGING_RESOURCE_UUIDS}"
if [ -z "$SELECTED_COOLIFY_API_URL" ] || [ -z "$SELECTED_COOLIFY_TOKEN" ] || [ -z "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" ]; then
echo "Coolify staging portal-domain-controller configuration is incomplete"
exit 1
fi
node scripts/coolify-portal-domain-controller.mjs set-release \
staging \
"$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" \
"$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/portal-domain-controller" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker" \
"$IMAGE_TAG"
STAGING_PORTAL_DOMAIN_CONTROLLER_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID")"
echo "STAGING_PORTAL_DOMAIN_CONTROLLER_DEPLOYMENT_UUID=$STAGING_PORTAL_DOMAIN_CONTROLLER_DEPLOYMENT_UUID" >> "$GITHUB_ENV"
node scripts/coolify-deploy.mjs wait \
"$STAGING_PORTAL_DOMAIN_CONTROLLER_DEPLOYMENT_UUID" \
"Coolify staging portal-domain-controller"
node scripts/coolify-portal-domain-controller.mjs assert-release \
staging \
"$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" \
"$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/portal-domain-controller" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker" \
"$IMAGE_TAG" \
"$STAGING_PORTAL_DOMAIN_CONTROLLER_DEPLOYMENT_UUID"
echo "STAGING_PORTAL_DOMAIN_CONTROLLER_RELEASE_ACCEPTED=true" >> "$GITHUB_ENV"
# This step is deliberately always-on. Once recovery is armed, any
# failure before the new controller release is accepted restores the
# validated previous release and resumes queued custom-domain work.
- name: Restore previous Coolify staging portal-domain-controller release on rollout failure
if: always()
env:
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_RESOURCE_UUIDS: ${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }}
COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }}
run: |
if [ "$DEPLOY_ACTION" != "stage" ]; then
echo "Staging portal-domain-controller recovery skipped for promote"
exit 0
fi
if [ "${STAGING_PORTAL_DOMAIN_CONTROLLER_RECOVERY_ARMED:-false}" != "true" ]; then
echo "Staging portal-domain-controller recovery skipped because it was not armed"
exit 0
fi
if [ "${STAGING_PORTAL_DOMAIN_CONTROLLER_RELEASE_ACCEPTED:-false}" = "true" ]; then
echo "Staging portal-domain-controller recovery skipped because the new release was accepted"
exit 0
fi
if [ -z "${STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION:-}" ]; then
echo "Staging portal-domain-controller previous release is missing"
exit 1
fi
SELECTED_COOLIFY_API_URL="$COOLIFY_STAGING_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_STAGING_TOKEN"
RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS="${STAGING_RESOLVED_RESOURCE_UUIDS:-$COOLIFY_STAGING_RESOURCE_UUIDS}"
node scripts/coolify-portal-domain-controller.mjs set-release \
staging \
"$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" \
"$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/portal-domain-controller" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker" \
"$STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION"
RECOVERY_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID")"
node scripts/coolify-deploy.mjs wait \
"$RECOVERY_DEPLOYMENT_UUID" \
"Coolify staging portal-domain-controller recovery"
node scripts/coolify-portal-domain-controller.mjs assert-release \
staging \
"$STAGING_PORTAL_DOMAIN_CONTROLLER_RESOURCE_UUID" \
"$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/portal-domain-controller" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker" \
"$STAGING_PORTAL_DOMAIN_CONTROLLER_PREVIOUS_REVISION" \
"$RECOVERY_DEPLOYMENT_UUID"
echo "Staging portal-domain-controller previous release restored after rollout failure"
- name: Validate Coolify production runtime preflight
env:
COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}
COOLIFY_RESOURCE_UUIDS: ${{ vars.COOLIFY_RESOURCE_UUIDS }}
COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }}
run: |
if [ "$DEPLOY_ACTION" != "promote" ]; then
echo "Coolify production inventory validation skipped for staging"
exit 0
fi
RESOLVED_RESOURCE_UUIDS="${PRODUCTION_RESOLVED_RESOURCE_UUIDS:-$COOLIFY_RESOURCE_UUIDS}"
WEB_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web"
SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN"
export SELECTED_COOLIFY_API_URL SELECTED_COOLIFY_TOKEN
node scripts/coolify-deploy.mjs assert-runtime-preflight \
"$RESOLVED_RESOURCE_UUIDS" \
"$WEB_IMAGE" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/hocuspocus" \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"
- name: Run production one-shot database migration
env:
MIGRATOR_DATABASE_URL: ${{ secrets.MIGRATOR_DATABASE_URL }}
run: |
if [ "$DEPLOY_ACTION" != "promote" ]; then
echo "Production one-shot migration skipped for staging"
exit 0
fi
if [ -z "${MIGRATOR_DATABASE_URL:-}" ]; then
echo "The production migrator database URL is required"
exit 1
fi
DIRECT_URL="$MIGRATOR_DATABASE_URL" \
docker run --rm --pull=always \
-e DIRECT_URL \
"$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web:$IMAGE_TAG" migrate
- name: Ensure Coolify production worker groups
env:
COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}
COOLIFY_RESOURCE_UUIDS: ${{ vars.COOLIFY_RESOURCE_UUIDS }}
COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }}
COOLIFY_PROVISION_WORKER_GROUPS: ${{ vars.COOLIFY_PROVISION_WORKER_GROUPS }}
COOLIFY_PROJECT_UUID: ${{ vars.COOLIFY_PROJECT_UUID }}
COOLIFY_SERVER_UUID: ${{ vars.COOLIFY_SERVER_UUID }}
COOLIFY_ENVIRONMENT_NAME: ${{ vars.COOLIFY_ENVIRONMENT_NAME }}
COOLIFY_ENVIRONMENT_UUID: ${{ vars.COOLIFY_ENVIRONMENT_UUID }}
COOLIFY_DESTINATION_UUID: ${{ vars.COOLIFY_DESTINATION_UUID }}
COOLIFY_WORKER_ENV_SOURCE_UUID: ${{ vars.COOLIFY_WORKER_ENV_SOURCE_UUID }}
run: |
if [ "$DEPLOY_ACTION" != "promote" ]; then
echo "Coolify production worker group provisioning skipped for staging"
exit 0
fi
node scripts/coolify-ensure-worker-groups.mjs production
- name: Configure Coolify production runtime resources
env:
COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}
COOLIFY_RESOURCE_UUIDS: ${{ vars.COOLIFY_RESOURCE_UUIDS }}
COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }}
run: |
if [ "$DEPLOY_ACTION" != "promote" ]; then
echo "Coolify production runtime configuration skipped for staging"
exit 0
fi
RESOLVED_COOLIFY_RESOURCE_UUIDS="${PRODUCTION_RESOLVED_RESOURCE_UUIDS:-$COOLIFY_RESOURCE_UUIDS}"
if [ -z "$COOLIFY_API_URL" ] || [ -z "$RESOLVED_COOLIFY_RESOURCE_UUIDS" ] || [ -z "$COOLIFY_TOKEN" ]; then
echo "Coolify production configuration is incomplete"
exit 1
fi
WEB_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web"
API_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api"
MCP_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp"
HOCUSPOCUS_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/hocuspocus"
WORKER_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"
APPS_JSON="$(mktemp)"
trap 'rm -f "${APPS_JSON:-}"' EXIT
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 \
--header "Authorization: Bearer $COOLIFY_TOKEN" \
"$COOLIFY_API_URL/applications" > "$APPS_JSON"
WEB_SURFACE_RESOURCE_UUIDS="$(SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL" SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN" node scripts/coolify-deploy.mjs resolve-web-surfaces "$RESOLVED_COOLIFY_RESOURCE_UUIDS" "$WEB_IMAGE")"
WEB_RESOURCE_UUID="${WEB_SURFACE_RESOURCE_UUIDS%%,*}"
REMAINING_WEB_SURFACE_RESOURCE_UUIDS="${WEB_SURFACE_RESOURCE_UUIDS#*,}"
ADMIN_RESOURCE_UUID="${REMAINING_WEB_SURFACE_RESOURCE_UUIDS%%,*}"
PORTAL_RESOURCE_UUID="${REMAINING_WEB_SURFACE_RESOURCE_UUIDS#*,}"
API_RESOURCE_UUID="$(API_IMAGE="$API_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.API_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
MCP_RESOURCE_UUID="$(MCP_IMAGE="$MCP_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.MCP_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
HOCUSPOCUS_RESOURCE_UUID="$(HOCUSPOCUS_IMAGE="$HOCUSPOCUS_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.HOCUSPOCUS_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
select_worker_resource() {
WORKER_QUEUE_GROUP="$1" WORKER_IMAGE="$WORKER_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const group = process.env.WORKER_QUEUE_GROUP || ""; const envPattern = new RegExp(`WORKER_QUEUE_GROUP\\s*(?:=|:)\\s*["'\''"]?${group}\\b`, "i"); const namePattern = new RegExp(`(?:^|[\\s/_-])worker[\\s_-]*${group}(?:$|[\\s:_-])`, "i"); const app = apps.find((candidate) => { const text = [candidate.name, candidate.description, candidate.docker_registry_image_name, candidate.fqdn, candidate.domain, candidate.domains, JSON.stringify(candidate.environment_variables || candidate.envs || candidate.environment || "")].filter(Boolean).join(" "); return candidate.docker_registry_image_name === process.env.WORKER_IMAGE && allowed.has(candidate.uuid) && (envPattern.test(text) || namePattern.test(text)); }); process.stdout.write(app?.uuid || "");' "$APPS_JSON"
}
WORKER_GENERAL_RESOURCE_UUID="$(select_worker_resource general)"
WORKER_MAIL_RESOURCE_UUID="$(select_worker_resource mail)"
WORKER_IMPORT_RESOURCE_UUID="$(select_worker_resource import)"
WORKER_WEBHOOK_RESOURCE_UUID="$(select_worker_resource webhook)"
WORKER_PLUGIN_RESOURCE_UUID="$(select_worker_resource plugin)"
WORKER_RESOURCE_UUIDS="$WORKER_GENERAL_RESOURCE_UUID,$WORKER_MAIL_RESOURCE_UUID,$WORKER_IMPORT_RESOURCE_UUID,$WORKER_WEBHOOK_RESOURCE_UUID,$WORKER_PLUGIN_RESOURCE_UUID"
if [ -z "$WEB_RESOURCE_UUID" ] || [ -z "$ADMIN_RESOURCE_UUID" ] || [ -z "$PORTAL_RESOURCE_UUID" ] || [ -z "$API_RESOURCE_UUID" ] || [ -z "$MCP_RESOURCE_UUID" ] || [ -z "$HOCUSPOCUS_RESOURCE_UUID" ] || [ -z "$WORKER_GENERAL_RESOURCE_UUID" ] || [ -z "$WORKER_MAIL_RESOURCE_UUID" ] || [ -z "$WORKER_IMPORT_RESOURCE_UUID" ] || [ -z "$WORKER_WEBHOOK_RESOURCE_UUID" ] || [ -z "$WORKER_PLUGIN_RESOURCE_UUID" ]; then
echo "Coolify production hivegrid-web, hivegrid-admin, hivegrid-portal, api, mcp, hocuspocus, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications must all exist"
exit 1
fi
for REQUIRED_RESOURCE_UUID in "$WEB_RESOURCE_UUID" "$ADMIN_RESOURCE_UUID" "$PORTAL_RESOURCE_UUID" "$API_RESOURCE_UUID" "$MCP_RESOURCE_UUID" "$HOCUSPOCUS_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
if ! printf ",%s," "$RESOLVED_COOLIFY_RESOURCE_UUIDS" | grep -Fq ",$REQUIRED_RESOURCE_UUID,"; then
echo "Coolify production deploy resources must include hivegrid-web, hivegrid-admin, hivegrid-portal, api, mcp, hocuspocus, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin"
exit 1
fi
done
node scripts/coolify-deploy.mjs assert-no-app-revision-override "$WEB_RESOURCE_UUID"
node scripts/coolify-deploy.mjs assert-no-app-revision-override "$ADMIN_RESOURCE_UUID"
node scripts/coolify-deploy.mjs assert-no-app-revision-override "$PORTAL_RESOURCE_UUID"
node scripts/coolify-deploy.mjs assert-no-app-revision-override "$API_RESOURCE_UUID"
node scripts/coolify-deploy.mjs assert-runtime-db-env "$WEB_RESOURCE_UUID" hivegrid_app web
node scripts/coolify-deploy.mjs assert-runtime-db-env "$ADMIN_RESOURCE_UUID" hivegrid_app admin
node scripts/coolify-deploy.mjs assert-runtime-db-env "$PORTAL_RESOURCE_UUID" hivegrid_portal portal
node scripts/coolify-deploy.mjs assert-runtime-db-env "$API_RESOURCE_UUID" hivegrid_app service
node scripts/coolify-deploy.mjs assert-runtime-db-env "$MCP_RESOURCE_UUID" hivegrid_app service
node scripts/coolify-deploy.mjs assert-runtime-db-env "$HOCUSPOCUS_RESOURCE_UUID" hivegrid_app service
for WORKER_RESOURCE_UUID in "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
node scripts/coolify-deploy.mjs assert-runtime-db-env "$WORKER_RESOURCE_UUID" hivegrid_worker service
done
node scripts/coolify-deploy.mjs assert-web-push-env "$WEB_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID"
DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$MCP_RESOURCE_UUID,$HOCUSPOCUS_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"
WEB_IMAGE_DIGEST="$(docker buildx imagetools inspect "$WEB_IMAGE:$IMAGE_TAG" --format '{{json .Manifest.Digest}}' | tr -d '"')"
if ! printf '%s' "$WEB_IMAGE_DIGEST" | grep -Eq '^sha256:[0-9a-f]{64}$'; then
echo "Production web image digest is missing or malformed"
exit 1
fi
node scripts/coolify-deploy.mjs set-web-surface-release \
"$WEB_RESOURCE_UUID" \
"$ADMIN_RESOURCE_UUID" \
"$PORTAL_RESOURCE_UUID" \
"$WEB_IMAGE" \
"$IMAGE_TAG" \
"$WEB_IMAGE_DIGEST"
{
echo "PRODUCTION_WEB_RESOURCE_UUID=$WEB_RESOURCE_UUID"
echo "PRODUCTION_ADMIN_RESOURCE_UUID=$ADMIN_RESOURCE_UUID"
echo "PRODUCTION_PORTAL_RESOURCE_UUID=$PORTAL_RESOURCE_UUID"
echo "PRODUCTION_WEB_IMAGE_DIGEST=$WEB_IMAGE_DIGEST"
echo "PRODUCTION_MCP_RESOURCE_UUID=$MCP_RESOURCE_UUID"
echo "PRODUCTION_HOCUSPOCUS_RESOURCE_UUID=$HOCUSPOCUS_RESOURCE_UUID"
echo "PRODUCTION_WORKER_RESOURCE_UUIDS=$WORKER_RESOURCE_UUIDS"
echo "PRODUCTION_WORKER_GENERAL_RESOURCE_UUID=$WORKER_GENERAL_RESOURCE_UUID"
echo "PRODUCTION_WORKER_MAIL_RESOURCE_UUID=$WORKER_MAIL_RESOURCE_UUID"
echo "PRODUCTION_WORKER_IMPORT_RESOURCE_UUID=$WORKER_IMPORT_RESOURCE_UUID"
echo "PRODUCTION_WORKER_WEBHOOK_RESOURCE_UUID=$WORKER_WEBHOOK_RESOURCE_UUID"
echo "PRODUCTION_WORKER_PLUGIN_RESOURCE_UUID=$WORKER_PLUGIN_RESOURCE_UUID"
echo "PRODUCTION_DEPENDENT_RESOURCE_UUIDS=$DEPENDENT_RESOURCE_UUIDS"
} >> "$GITHUB_ENV"
IMAGE_TAG_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ docker_registry_image_tag: process.env.IMAGE_TAG }))')"
for RESOURCE_UUID in "$API_RESOURCE_UUID" "$MCP_RESOURCE_UUID" "$HOCUSPOCUS_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 --request PATCH \
--header "Authorization: Bearer $COOLIFY_TOKEN" \
--header "Content-Type: application/json" \
--data "$IMAGE_TAG_PAYLOAD" \
"$COOLIFY_API_URL/applications/$RESOURCE_UUID" >/dev/null
done
- name: Promote staging images to production aliases
run: |
if [ "$DEPLOY_ACTION" != "promote" ]; then
echo "Promotion skipped for staging"
exit 0
fi
for TARGET in web api worker mcp hocuspocus portal-domain-controller; do
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
docker buildx imagetools create --prefer-index=false --tag "$IMAGE:main" "$IMAGE:$IMAGE_TAG"
done
- name: Trigger Coolify production deploy
env:
COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}
COOLIFY_RESOURCE_UUIDS: ${{ vars.COOLIFY_RESOURCE_UUIDS }}
COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }}
run: |
if [ "$DEPLOY_ACTION" != "promote" ]; then
echo "Production deploy skipped for staging"
exit 0
fi
if [ -z "$COOLIFY_API_URL" ] || [ -z "$PRODUCTION_WEB_RESOURCE_UUID" ] || [ -z "$PRODUCTION_ADMIN_RESOURCE_UUID" ] || [ -z "$PRODUCTION_PORTAL_RESOURCE_UUID" ] || [ -z "$COOLIFY_TOKEN" ]; then
echo "Coolify production configuration is incomplete"
exit 1
fi
SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN"
PRODUCTION_WEB_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_WEB_RESOURCE_UUID")"
echo "PRODUCTION_WEB_DEPLOYMENT_UUID=$PRODUCTION_WEB_DEPLOYMENT_UUID" >> "$GITHUB_ENV"
node scripts/coolify-deploy.mjs wait \
"$PRODUCTION_WEB_DEPLOYMENT_UUID" \
"Coolify production hivegrid-web"
PRODUCTION_ADMIN_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_ADMIN_RESOURCE_UUID")"
echo "PRODUCTION_ADMIN_DEPLOYMENT_UUID=$PRODUCTION_ADMIN_DEPLOYMENT_UUID" >> "$GITHUB_ENV"
node scripts/coolify-deploy.mjs wait \
"$PRODUCTION_ADMIN_DEPLOYMENT_UUID" \
"Coolify production hivegrid-admin"
PRODUCTION_PORTAL_DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$PRODUCTION_PORTAL_RESOURCE_UUID")"
echo "PRODUCTION_PORTAL_DEPLOYMENT_UUID=$PRODUCTION_PORTAL_DEPLOYMENT_UUID" >> "$GITHUB_ENV"
node scripts/coolify-deploy.mjs wait \
"$PRODUCTION_PORTAL_DEPLOYMENT_UUID" \
"Coolify production hivegrid-portal"
- name: Wait for Coolify production health
env:
COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}
COOLIFY_RESOURCE_UUIDS: ${{ vars.COOLIFY_RESOURCE_UUIDS }}
COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }}
run: |
if [ "$DEPLOY_ACTION" != "promote" ]; then
echo "Production health wait skipped for staging"
exit 0
fi
if [ -z "$COOLIFY_API_URL" ] || [ -z "$PRODUCTION_WEB_RESOURCE_UUID" ] || [ -z "$PRODUCTION_ADMIN_RESOURCE_UUID" ] || [ -z "$PRODUCTION_PORTAL_RESOURCE_UUID" ] || [ -z "$PRODUCTION_WEB_IMAGE_DIGEST" ] || [ -z "$COOLIFY_TOKEN" ]; then
echo "Coolify production configuration is incomplete"
exit 1
fi
WEB_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web"
EXPECTED_WEB_DIGEST_TAG="sha256-${PRODUCTION_WEB_IMAGE_DIGEST#sha256:}"
SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN"
if [ -z "${PRODUCTION_WEB_DEPLOYMENT_UUID:-}" ] || [ -z "${PRODUCTION_PORTAL_DEPLOYMENT_UUID:-}" ]; then
echo "Coolify production web surface deployment UUIDs are missing"
exit 1
fi
verify_live_revision() {
APP_JSON="$1"
LABEL="$2"
REQUIRE_HEALTH_URL="${3:-auto}"
EXPECTED_IMAGE_ALIAS="${4:-main}"
EXPECTED_SERVICE="${5:-}"
EXPECTED_SURFACE="${6:-}"
HEALTH_URL="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const raw = app.fqdn || app.domain || app.domains || ""; const value = Array.isArray(raw) ? raw[0] : String(raw).split(/[,\s]+/).find(Boolean); process.stdout.write(value ? value.replace(/\/+$/, "") : "");' "$APP_JSON")"
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api|mcp)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
if [ -z "$HEALTH_URL" ]; then
if [ "$REQUIRE_HEALTH_URL" = "required" ] || [ "$HTTP_RESOURCE" = "true" ]; then
echo "$LABEL is an HTTP resource but has no health URL"
exit 1
fi
CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")"
if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then
echo "$LABEL non-HTTP resource image tag does not match expected alias"
exit 1
fi
echo "$LABEL non-HTTP resource image tag matches expected alias"
return
fi
HEALTH_JSON="$(mktemp)"
READINESS_JSON="$(mktemp)"
if ! curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL/healthz" > "$HEALTH_JSON"; then
rm -f "$HEALTH_JSON" "$READINESS_JSON"
echo "$LABEL health endpoint is not ready yet"
return 1
fi
if ! curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL/readyz" > "$READINESS_JSON"; then
rm -f "$HEALTH_JSON" "$READINESS_JSON"
echo "$LABEL readiness endpoint is not ready yet"
return 1
fi
HEALTH_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$HEALTH_JSON")"
HEALTH_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$HEALTH_JSON")"
HEALTH_SURFACE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.surface || "");' "$HEALTH_JSON")"
READINESS_STATUS="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.status || "");' "$READINESS_JSON")"
READINESS_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$READINESS_JSON")"
READINESS_SURFACE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.surface || "");' "$READINESS_JSON")"
READINESS_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$READINESS_JSON")"
rm -f "$HEALTH_JSON" "$READINESS_JSON"
if [ -n "$EXPECTED_SERVICE" ] && [ "$HEALTH_SERVICE" != "$EXPECTED_SERVICE" ]; then
echo "$LABEL health service does not match expected service"
exit 1
fi
if [ -n "$EXPECTED_SURFACE" ] && [ "$HEALTH_SURFACE" != "$EXPECTED_SURFACE" ]; then
echo "$LABEL health surface does not match expected surface"
exit 1
fi
if [ "$HEALTH_REVISION" != "$IMAGE_TAG" ]; then
echo "$LABEL health revision does not match promoted image yet"
return 1
fi
if [ "$READINESS_STATUS" != "ready" ]; then
echo "$LABEL readiness status is not ready yet"
return 1
fi
if [ "$READINESS_REVISION" != "$IMAGE_TAG" ]; then
echo "$LABEL readiness revision does not match promoted image yet"
return 1
fi
if [ -n "$EXPECTED_SERVICE" ] && [ "$READINESS_SERVICE" != "$EXPECTED_SERVICE" ]; then
echo "$LABEL readiness service does not match expected service"
exit 1
fi
if [ -n "$EXPECTED_SURFACE" ] && [ "$READINESS_SURFACE" != "$EXPECTED_SURFACE" ]; then
echo "$LABEL readiness surface does not match expected surface"
exit 1
fi
}
RESOURCE_INDEX=0
for SURFACE in core admin portal; do
RESOURCE_INDEX=$((RESOURCE_INDEX + 1))
if [ "$SURFACE" = "core" ]; then
RESOURCE_UUID="$PRODUCTION_WEB_RESOURCE_UUID"
EXPECTED_SERVICE="hivegrid-web"
elif [ "$SURFACE" = "admin" ]; then
RESOURCE_UUID="$PRODUCTION_ADMIN_RESOURCE_UUID"
EXPECTED_SERVICE="hivegrid-admin"
else
RESOURCE_UUID="$PRODUCTION_PORTAL_RESOURCE_UUID"
EXPECTED_SERVICE="hivegrid-portal"
fi
HEALTHY=false
for ATTEMPT in $(seq 1 24); do
APP_JSON="$(mktemp)"
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 \
--header "Authorization: Bearer $COOLIFY_TOKEN" \
"$COOLIFY_API_URL/applications/$RESOURCE_UUID" > "$APP_JSON"
STATUS="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.status || "unknown");' "$APP_JSON")"
CONFIGURED_IMAGE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_name || "");' "$APP_JSON")"
CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || "");' "$APP_JSON")"
echo "Coolify production resource #$RESOURCE_INDEX status: $STATUS"
if [ "$CONFIGURED_IMAGE" != "$WEB_IMAGE" ] || [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_WEB_DIGEST_TAG" ]; then
rm -f "$APP_JSON"
echo "Coolify production web surface #$RESOURCE_INDEX is not pinned to the paired registry digest"
exit 1
fi
if [ "$STATUS" = "running:healthy" ]; then
if verify_live_revision "$APP_JSON" "Coolify production resource #$RESOURCE_INDEX" required "$IMAGE_TAG" "$EXPECTED_SERVICE" "$SURFACE"; then
rm -f "$APP_JSON"
HEALTHY=true
break
fi
fi
rm -f "$APP_JSON"
if printf "%s" "$STATUS" | grep -Eiq 'failed|exited|dead'; then
echo "Coolify production resource #$RESOURCE_INDEX entered a terminal failure status"
exit 1
fi
sleep 5
done
if [ "$HEALTHY" != "true" ]; then
echo "Coolify production resource #$RESOURCE_INDEX did not become healthy in time"
exit 1
fi
done
ROLLED_OUT_WEB_IMAGE_DIGEST="$(docker buildx imagetools inspect "$WEB_IMAGE:$IMAGE_TAG" --format '{{json .Manifest.Digest}}' | tr -d '"')"
if [ "$ROLLED_OUT_WEB_IMAGE_DIGEST" != "$PRODUCTION_WEB_IMAGE_DIGEST" ]; then
echo "Production web image digest changed during rollout"
exit 1
fi
node scripts/coolify-deploy.mjs assert-web-surface-release \
"$PRODUCTION_WEB_RESOURCE_UUID" \
"$PRODUCTION_ADMIN_RESOURCE_UUID" \
"$PRODUCTION_PORTAL_RESOURCE_UUID" \
"$WEB_IMAGE" \
"$IMAGE_TAG" \
"$ROLLED_OUT_WEB_IMAGE_DIGEST" \
"$PRODUCTION_WEB_DEPLOYMENT_UUID" \
"$PRODUCTION_ADMIN_DEPLOYMENT_UUID" \
"$PRODUCTION_PORTAL_DEPLOYMENT_UUID"
- name: Verify production migration status after web deploy
if: env.DEPLOY_ACTION == 'promote'
env:
DATABASE_URL: ${{ secrets.MIGRATOR_DATABASE_URL }}
run: |
if [ -z "$DATABASE_URL" ]; then
echo "DATABASE_URL is required for production migration status verification"
exit 1
fi
pnpm run verify:db-migrate-status
- name: Trigger Coolify production dependent deploys
env:
COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}
COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }}
run: |
if [ "$DEPLOY_ACTION" != "promote" ]; then
echo "Production dependent deploy skipped for staging"
exit 0
fi
if [ -z "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS" ]; then
echo "No production dependent resources to deploy"
exit 0
fi
if [ -z "$COOLIFY_API_URL" ] || [ -z "$COOLIFY_TOKEN" ]; then
echo "Coolify production configuration is incomplete"
exit 1
fi
SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN"
node scripts/coolify-deploy.mjs configure-mcp-healthcheck "$PRODUCTION_MCP_RESOURCE_UUID" "$IMAGE_TAG"
PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS=""
PRODUCTION_DEPENDENT_TRIGGER_FAILED=false
DEPLOYMENT_INDEX=0
for RESOURCE_UUID in $(printf "%s" "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do
DEPLOYMENT_INDEX=$((DEPLOYMENT_INDEX + 1))
if DEPLOYMENT_UUID="$(node scripts/coolify-deploy.mjs trigger "$RESOURCE_UUID")"; then
PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS="${PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS:+$PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS,}$DEPLOYMENT_UUID"
# Coolify can dispatch the same queued deployment twice when several
# resources are enqueued in one burst. Keep deployments serialized;
# the following health/revision probes still fan out independently.
if ! node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify production dependent deploy #$DEPLOYMENT_INDEX"; then
echo "A serialized Coolify production dependent deploy failed"
PRODUCTION_DEPENDENT_TRIGGER_FAILED=true
fi
else
echo "A Coolify production dependent deploy trigger failed"
PRODUCTION_DEPENDENT_TRIGGER_FAILED=true
fi
done
echo "PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS=$PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS" >> "$GITHUB_ENV"
echo "PRODUCTION_DEPENDENT_TRIGGER_FAILED=$PRODUCTION_DEPENDENT_TRIGGER_FAILED" >> "$GITHUB_ENV"
- name: Wait for Coolify production dependent health
env:
COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}
COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }}
run: |
if [ "$DEPLOY_ACTION" != "promote" ]; then
echo "Production dependent health wait skipped for staging"
exit 0
fi
if [ -z "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS" ]; then
echo "No production dependent resources to wait for"
exit 0
fi
if [ -z "$PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS" ] && [ "${PRODUCTION_DEPENDENT_TRIGGER_FAILED:-false}" != "true" ]; then
echo "Coolify production dependent deployment UUIDs are missing"
exit 1
fi
if [ -z "$COOLIFY_API_URL" ] || [ -z "$COOLIFY_TOKEN" ]; then
echo "Coolify production configuration is incomplete"
exit 1
fi
SELECTED_COOLIFY_API_URL="$COOLIFY_API_URL"
SELECTED_COOLIFY_TOKEN="$COOLIFY_TOKEN"
DEPLOYMENT_INDEX=0
DEPLOYMENT_WAIT_PIDS=""
for DEPLOYMENT_UUID in $(printf "%s" "$PRODUCTION_DEPENDENT_DEPLOYMENT_UUIDS" | tr ',' ' '); do
DEPLOYMENT_INDEX=$((DEPLOYMENT_INDEX + 1))
node scripts/coolify-deploy.mjs wait "$DEPLOYMENT_UUID" "Coolify production dependent #$DEPLOYMENT_INDEX" &
DEPLOYMENT_WAIT_PIDS="$DEPLOYMENT_WAIT_PIDS $!"
done
DEPLOYMENT_WAIT_FAILED=false
for PID in $DEPLOYMENT_WAIT_PIDS; do
if ! wait "$PID"; then
DEPLOYMENT_WAIT_FAILED=true
fi
done
if [ "$DEPLOYMENT_WAIT_FAILED" = "true" ]; then
echo "One or more Coolify production dependent deployment waits failed"
exit 1
fi
verify_live_revision() {
APP_JSON="$1"
LABEL="$2"
REQUIRE_HEALTH_URL="${3:-auto}"
EXPECTED_IMAGE_ALIAS="${4:-main}"
EXPECTED_SERVICE="${5:-}"
RESOURCE_KIND="${6:-auto}"
EXPECTED_WORKER_GROUP="${7:-}"
HEALTH_URL="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const raw = app.fqdn || app.domain || app.domains || ""; const value = Array.isArray(raw) ? raw[0] : String(raw).split(/[,\s]+/).find(Boolean); process.stdout.write(value ? value.replace(/\/+$/, "") : "");' "$APP_JSON")"
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api|mcp)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
WORKER_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])worker(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
if [ "$RESOURCE_KIND" = "hocuspocus" ]; then
CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")"
if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then
echo "$LABEL Hocuspocus image tag does not match expected revision"
exit 1
fi
echo "$LABEL Hocuspocus Docker health and image revision are healthy"
return
fi
if [ "$RESOURCE_KIND" = "worker" ] || [ "$WORKER_RESOURCE" = "true" ]; then
CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")"
if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then
echo "$LABEL worker image tag does not match expected alias"
exit 1
fi
if [ -z "$EXPECTED_WORKER_GROUP" ]; then
echo "$LABEL worker group could not be resolved"
exit 1
fi
EXPECTED_HEALTH_COMMAND="node apps/worker/dist/deploy-health.js $IMAGE_TAG $EXPECTED_WORKER_GROUP"
CONFIGURED_HEALTH_COMMAND="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.health_check_command || "");' "$APP_JSON")"
CONFIGURED_HEALTH_TYPE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.health_check_type || "");' "$APP_JSON")"
HEALTH_CHECK_ENABLED="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = app.health_check_enabled; process.stdout.write(value === true || value === "true" || value === 1 || value === "1" ? "true" : "false");' "$APP_JSON")"
CUSTOM_HEALTHCHECK_FOUND="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = app.custom_healthcheck_found; process.stdout.write(value === false || value === "false" || value === 0 || value === "0" ? "false" : value === true || value === "true" || value === 1 || value === "1" ? "true" : "unknown");' "$APP_JSON")"
if [ "$HEALTH_CHECK_ENABLED" != "true" ] || [ "$CONFIGURED_HEALTH_TYPE" != "cmd" ] || [ "$CONFIGURED_HEALTH_COMMAND" != "$EXPECTED_HEALTH_COMMAND" ] || [ "$CUSTOM_HEALTHCHECK_FOUND" != "false" ]; then
echo "$LABEL worker healthcheck is not bound to the deployed revision"
exit 1
fi
echo "$LABEL worker Coolify status and revision-bound command healthcheck are healthy"
return
fi
if [ -z "$HEALTH_URL" ]; then
if [ "$REQUIRE_HEALTH_URL" = "required" ] || [ "$HTTP_RESOURCE" = "true" ]; then
echo "$LABEL is an HTTP resource but has no health URL"
exit 1
fi
CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")"
if [ "$CONFIGURED_IMAGE_TAG" != "$EXPECTED_IMAGE_ALIAS" ]; then
echo "$LABEL non-HTTP resource image tag does not match expected alias"
exit 1
fi
echo "$LABEL non-HTTP resource image tag matches expected alias"
return
fi
HEALTH_PATH="/healthz"
if [ "$RESOURCE_KIND" = "mcp" ]; then
HEALTH_PATH="/readyz"
EXPECTED_MCP_HEALTH_COMMAND="node apps/mcp/dist/deploy-health.js $IMAGE_TAG"
CONFIGURED_MCP_HEALTH_COMMAND="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.health_check_command || "");' "$APP_JSON")"
CONFIGURED_MCP_HEALTH_TYPE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.health_check_type || "");' "$APP_JSON")"
MCP_HEALTH_CHECK_ENABLED="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = app.health_check_enabled; process.stdout.write(value === true || value === "true" || value === 1 || value === "1" ? "true" : "false");' "$APP_JSON")"
MCP_CUSTOM_HEALTHCHECK_FOUND="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = app.custom_healthcheck_found; process.stdout.write(value === false || value === "false" || value === 0 || value === "0" ? "false" : value === true || value === "true" || value === 1 || value === "1" ? "true" : "unknown");' "$APP_JSON")"
if [ "$MCP_HEALTH_CHECK_ENABLED" != "true" ] || [ "$CONFIGURED_MCP_HEALTH_TYPE" != "cmd" ] || [ "$CONFIGURED_MCP_HEALTH_COMMAND" != "$EXPECTED_MCP_HEALTH_COMMAND" ] || [ "$MCP_CUSTOM_HEALTHCHECK_FOUND" != "false" ]; then
echo "$LABEL MCP healthcheck is not bound to the deployed revision"
exit 1
fi
fi
HEALTH_JSON="$(mktemp)"
if ! curl --fail --silent --show-error --connect-timeout 10 --max-time 30 "$HEALTH_URL$HEALTH_PATH" > "$HEALTH_JSON"; then
rm -f "$HEALTH_JSON"
echo "$LABEL readiness endpoint is not ready yet"
return 1
fi
HEALTH_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$HEALTH_JSON")"
HEALTH_SERVICE="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.service || "");' "$HEALTH_JSON")"
rm -f "$HEALTH_JSON"
if [ -n "$EXPECTED_SERVICE" ] && [ "$HEALTH_SERVICE" != "$EXPECTED_SERVICE" ]; then
echo "$LABEL health service does not match expected service"
exit 1
fi
if [ "$HEALTH_REVISION" != "$IMAGE_TAG" ]; then
echo "$LABEL health revision does not match promoted image yet"
return 1
fi
}
check_dependent_resource() {
RESOURCE_UUID="$1"
RESOURCE_INDEX="$2"
HEALTHY=false
for ATTEMPT in $(seq 1 24); do
APP_JSON="$(mktemp)"
curl --fail --silent --show-error --connect-timeout 10 --max-time 30 \
--header "Authorization: Bearer $COOLIFY_TOKEN" \
"$COOLIFY_API_URL/applications/$RESOURCE_UUID" > "$APP_JSON"
STATUS="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.status || "unknown");' "$APP_JSON")"
echo "Coolify production dependent resource #$RESOURCE_INDEX status: $STATUS"
if [ "$STATUS" = "running:healthy" ]; then
RESOURCE_KIND="api"
EXPECTED_SERVICE="api"
EXPECTED_WORKER_GROUP=""
if [ "$RESOURCE_UUID" = "$PRODUCTION_MCP_RESOURCE_UUID" ]; then
RESOURCE_KIND="mcp"
EXPECTED_SERVICE="mcp"
fi
if [ "$RESOURCE_UUID" = "$PRODUCTION_HOCUSPOCUS_RESOURCE_UUID" ]; then
RESOURCE_KIND="hocuspocus"
EXPECTED_SERVICE=""
fi
if printf ",%s," "$PRODUCTION_WORKER_RESOURCE_UUIDS" | grep -Fq ",$RESOURCE_UUID,"; then
RESOURCE_KIND="worker"
EXPECTED_SERVICE=""
case "$RESOURCE_UUID" in
"$PRODUCTION_WORKER_GENERAL_RESOURCE_UUID") EXPECTED_WORKER_GROUP="general" ;;
"$PRODUCTION_WORKER_MAIL_RESOURCE_UUID") EXPECTED_WORKER_GROUP="mail" ;;
"$PRODUCTION_WORKER_IMPORT_RESOURCE_UUID") EXPECTED_WORKER_GROUP="import" ;;
"$PRODUCTION_WORKER_WEBHOOK_RESOURCE_UUID") EXPECTED_WORKER_GROUP="webhook" ;;
"$PRODUCTION_WORKER_PLUGIN_RESOURCE_UUID") EXPECTED_WORKER_GROUP="plugin" ;;
esac
fi
if verify_live_revision "$APP_JSON" "Coolify production dependent resource #$RESOURCE_INDEX" auto "$IMAGE_TAG" "$EXPECTED_SERVICE" "$RESOURCE_KIND" "$EXPECTED_WORKER_GROUP"; then
rm -f "$APP_JSON"
HEALTHY=true
break
fi
fi
rm -f "$APP_JSON"
if printf "%s" "$STATUS" | grep -Eiq 'failed|exited|dead'; then
echo "Coolify production dependent resource #$RESOURCE_INDEX entered a terminal failure status"
exit 1
fi
sleep 5
done
if [ "$HEALTHY" != "true" ]; then
echo "Coolify production dependent resource #$RESOURCE_INDEX did not become healthy in time"
exit 1
fi
}
RESOURCE_INDEX=0
RESOURCE_WAIT_PIDS=""
for RESOURCE_UUID in $(printf "%s" "$PRODUCTION_DEPENDENT_RESOURCE_UUIDS" | tr ',' ' '); do
RESOURCE_INDEX=$((RESOURCE_INDEX + 1))
check_dependent_resource "$RESOURCE_UUID" "$RESOURCE_INDEX" &
RESOURCE_WAIT_PIDS="$RESOURCE_WAIT_PIDS $!"
done
RESOURCE_WAIT_FAILED=false
for PID in $RESOURCE_WAIT_PIDS; do
if ! wait "$PID"; then
RESOURCE_WAIT_FAILED=true
fi
done
if [ "$RESOURCE_WAIT_FAILED" = "true" ]; then
echo "One or more Coolify production dependent resources failed health checks"
exit 1
fi
if [ "${PRODUCTION_DEPENDENT_TRIGGER_FAILED:-false}" = "true" ]; then
echo "One or more Coolify production dependent deploy triggers failed"
exit 1
fi
- name: Mark Sentry deploy
if: success()
env:
SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
SENTRY_URL: ${{ secrets.SENTRY_URL }}
run: |
if [ -z "${SENTRY_AUTH_TOKEN:-}" ] || [ -z "${SENTRY_ORG:-}" ] || [ -z "${SENTRY_URL:-}" ]; then
echo "Sentry secrets not configured; skipping deploy marker"
exit 0
fi
if [ "$DEPLOY_ACTION" = "promote" ]; then
SENTRY_ENVIRONMENT=production
RELEASE_SHA="${IMAGE_TAG}"
else
SENTRY_ENVIRONMENT=staging
RELEASE_SHA="${SHA}"
fi
for PROJECT in api worker web; do
bash scripts/ci/sentry-release.sh \
--project "$PROJECT" \
--environment "$SENTRY_ENVIRONMENT" \
--release "$RELEASE_SHA" \
--sourcemaps none
done
- name: Purge asset CDN
# Whole-zone purge against Bunny.net so the edge picks up the new
# hash-named bundles immediately. The helper script no-ops when
# BUNNY_API_KEY is empty, so this step is safe to wire even before
# the operator has provisioned a Bunny pull zone.
# See docs/ops/asset-cdn.md for the one-time setup.
if: success()
env:
BUNNY_PULL_ZONE_ID: ${{ secrets.BUNNY_PULL_ZONE_ID }}
BUNNY_API_KEY: ${{ secrets.BUNNY_API_KEY }}
ASSET_CDN_URL: ${{ vars.ASSET_CDN_URL }}
run: |
if [ -z "${BUNNY_API_KEY:-}" ] || [ -z "${BUNNY_PULL_ZONE_ID:-}" ]; then
echo "BUNNY_API_KEY / BUNNY_PULL_ZONE_ID not configured; skipping CDN purge"
exit 0
fi
pnpm exec tsx scripts/cdn/bunny-purge.ts