Update hivegrid-cloud reference to latest main

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
davidandClaude Fable 5 committed 2026-07-08 16:15:37 +02:00
1 parent e98d38e716
commit 7b21524c86
3781 files changed
+373484 -225607

No files matched your search

@@ -3,6 +3,20 @@ name: Deploy
on:
push:
branches: [main]
# Skip the staging deploy for test-/docs-only merges (e.g. coverage waves, doc updates).
# paths-ignore skips the run only when EVERY changed file matches a pattern; any app or
# config source change still deploys. Manual deploys via workflow_dispatch are unaffected.
paths-ignore:
- "**/*.test.ts"
- "**/*.test.tsx"
- "**/*.test.mjs"
- "**/*.test.js"
- "**/*.spec.ts"
- "docs/**"
- ".planning/**"
- "**/*.md"
- ".gitea/workflows/**"
- ".github/workflows/**"
workflow_dispatch:
inputs:
action:
@@ -18,10 +32,10 @@ on:
default: "false"
env:
TOOLCTRL_GITEA_HOST_IP: ${{ vars.TOOLCTRL_GITEA_HOST_IP }}
HIVEGRID_GITEA_HOST_IP: ${{ vars.HIVEGRID_GITEA_HOST_IP }}
concurrency:
group: boilerplate-deploy
group: hivegrid-deploy
cancel-in-progress: false
jobs:
@@ -32,12 +46,12 @@ jobs:
steps:
- name: Resolve internal Gitea host
run: |
if [ -n "${TOOLCTRL_GITEA_HOST_IP:-}" ]; then
if ! TOOLCTRL_GITEA_HOST_IP="$TOOLCTRL_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.TOOLCTRL_GITEA_HOST_IP || "") ? 0 : 1)'; then
echo "TOOLCTRL_GITEA_HOST_IP must be a single IPv4 or IPv6 token"
if [ -n "${HIVEGRID_GITEA_HOST_IP:-}" ]; then
if ! HIVEGRID_GITEA_HOST_IP="$HIVEGRID_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.HIVEGRID_GITEA_HOST_IP || "") ? 0 : 1)'; then
echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token"
exit 1
fi
printf '%s git.tool-ctrl.de\n' "$TOOLCTRL_GITEA_HOST_IP" >> /etc/hosts
printf '%s git.tool-ctrl.de\n' "$HIVEGRID_GITEA_HOST_IP" >> /etc/hosts
fi
- name: Checkout code
@@ -120,12 +134,12 @@ jobs:
- name: Setup hosts override and Docker CLI
run: |
if [ -n "${TOOLCTRL_GITEA_HOST_IP:-}" ]; then
if ! TOOLCTRL_GITEA_HOST_IP="$TOOLCTRL_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.TOOLCTRL_GITEA_HOST_IP || "") ? 0 : 1)'; then
echo "TOOLCTRL_GITEA_HOST_IP must be a single IPv4 or IPv6 token"
if [ -n "${HIVEGRID_GITEA_HOST_IP:-}" ]; then
if ! HIVEGRID_GITEA_HOST_IP="$HIVEGRID_GITEA_HOST_IP" node -e 'process.exit(require("node:net").isIP(process.env.HIVEGRID_GITEA_HOST_IP || "") ? 0 : 1)'; then
echo "HIVEGRID_GITEA_HOST_IP must be a single IPv4 or IPv6 token"
exit 1
fi
printf '%s git.tool-ctrl.de\n' "$TOOLCTRL_GITEA_HOST_IP" >> /etc/hosts
printf '%s git.tool-ctrl.de\n' "$HIVEGRID_GITEA_HOST_IP" >> /etc/hosts
fi
MISSING_TOOLS=""
@@ -191,7 +205,7 @@ jobs:
git cat-file -e "$IMAGE_TAG^{commit}"
git merge-base --is-ancestor "$IMAGE_TAG" origin/main
for TARGET in web api worker; do
for TARGET in web api worker mcp; do
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
SHA_DIGEST="$(docker buildx imagetools inspect "$IMAGE:$IMAGE_TAG" --format '{{json .Manifest.Digest}}')"
STAGING_DIGEST="$(docker buildx imagetools inspect "$IMAGE:staging" --format '{{json .Manifest.Digest}}')"
@@ -215,7 +229,7 @@ jobs:
cd "$GITHUB_WORKSPACE"
BUILD_PIDS=()
BUILD_TARGETS=()
for TARGET in web api worker; do
for TARGET in web api worker mcp; do
(
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
CACHE_REF="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/build-cache:$TARGET"
@@ -229,22 +243,28 @@ jobs:
--build-arg VITE_SENTRY_ENVIRONMENT="$VITE_SENTRY_ENVIRONMENT"
--build-arg VITE_SENTRY_RELEASE="$SHA"
--build-arg VITE_OTEL_EXPORTER_OTLP_ENDPOINT="$VITE_OTEL_EXPORTER_OTLP_ENDPOINT"
--build-arg VITE_OTEL_SERVICE_NAME="boilerplate-web"
--build-arg VITE_OTEL_SERVICE_NAME="hivegrid-web"
)
fi
# `ignore-error=true` on the cache export: writing the build cache
# to the registry is an optimization, not part of producing/pushing
# the image. The Gitea registry rate-limits (HTTP 429) build-cache
# blob writes under bursts of back-to-back deploys (made worse by
# web/api/worker exporting to the same cache in parallel); without
# this flag a 429 on a cache layer fails the whole build even though
# the image itself built and pushed fine.
docker buildx build --push --target "$TARGET" --file Dockerfile \
--cache-from "type=registry,ref=$CACHE_REF" \
--cache-to "type=registry,ref=$CACHE_REF,mode=max,ignore-error=true" \
--build-arg APP_REVISION="$SHA" \
"${WEB_BUILD_ARGS[@]}" \
--tag "$IMAGE:$IMAGE_TAG" .
# The registry cache is read-only in CI. Exporting build-cache blobs
# to Gitea is optional and has repeatedly failed staging deploys with
# registry-side 499/context deadline errors after the image itself
# built successfully.
for ATTEMPT in 1 2 3; do
if docker buildx build --push --target "$TARGET" --provenance=false --file Dockerfile \
--cache-from "type=registry,ref=$CACHE_REF" \
--build-arg APP_REVISION="$SHA" \
"${WEB_BUILD_ARGS[@]}" \
--tag "$IMAGE:$IMAGE_TAG" .; then
exit 0
fi
if [ "$ATTEMPT" -lt 3 ]; then
echo "Docker build/push failed for $TARGET on attempt $ATTEMPT; retrying"
sleep $((ATTEMPT * 20))
fi
done
echo "Docker build/push failed for $TARGET after 3 attempts"
exit 1
) &
BUILD_PIDS+=("$!")
BUILD_TARGETS+=("$TARGET")
@@ -275,7 +295,7 @@ jobs:
echo "Cosign signing skipped: COSIGN_PRIVATE_KEY or COSIGN_PASSWORD not configured"
exit 0
fi
for TARGET in web api worker; do
for TARGET in web api worker mcp; do
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
cosign sign --yes --key env://COSIGN_PRIVATE_KEY "$IMAGE:$IMAGE_TAG"
done
@@ -300,16 +320,19 @@ jobs:
node scripts/coolify-ensure-worker-groups.mjs staging
- name: Configure Coolify staging migration command
# Migrations and the guarded seed run inside the web container entrypoint
# (start.sh, gated by MIGRATE_ON_BOOT/SEED_ON_BOOT) because Coolify's
# pre/post_deployment_command never fired for docker-image applications.
# This step upserts the boot flags on the web resource and clears the
# dead hook slots so they are not mistaken for an active mechanism.
- name: Configure Coolify staging migration boot flags
env:
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_RESOURCE_UUIDS: ${{ vars.COOLIFY_STAGING_RESOURCE_UUIDS }}
COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }}
MIGRATION_COMMAND: cd /myapp && pnpm exec prisma migrate deploy --config prisma.config.ts
SEED_COMMAND: cd /myapp && SEED_DEMO_DATA=true ALLOW_STAGING_DEMO_SEED=true NODE_ENV=production node packages/db/dist/production-seed.server.js
run: |
if [ "$DEPLOY_ACTION" != "stage" ]; then
echo "Coolify migration command configuration skipped for promote"
echo "Coolify migration boot flag configuration skipped for promote"
exit 0
fi
@@ -323,6 +346,7 @@ jobs:
WEB_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web"
API_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api"
MCP_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp"
WORKER_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"
APPS_JSON="$(mktemp)"
trap 'rm -f "${APPS_JSON:-}"' EXIT
@@ -331,6 +355,7 @@ jobs:
"$SELECTED_COOLIFY_API_URL/applications" > "$APPS_JSON"
WEB_RESOURCE_UUID="$(WEB_IMAGE="$WEB_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.WEB_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
API_RESOURCE_UUID="$(API_IMAGE="$API_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.API_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
MCP_RESOURCE_UUID="$(MCP_IMAGE="$MCP_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.MCP_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
select_worker_resource() {
WORKER_QUEUE_GROUP="$1" WORKER_IMAGE="$WORKER_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const group = process.env.WORKER_QUEUE_GROUP || ""; const envPattern = new RegExp(`WORKER_QUEUE_GROUP\\s*(?:=|:)\\s*["'\''"]?${group}\\b`, "i"); const namePattern = new RegExp(`(?:^|[\\s/_-])worker[\\s_-]*${group}(?:$|[\\s:_-])`, "i"); const app = apps.find((candidate) => { const text = [candidate.name, candidate.description, candidate.docker_registry_image_name, candidate.fqdn, candidate.domain, candidate.domains, JSON.stringify(candidate.environment_variables || candidate.envs || candidate.environment || "")].filter(Boolean).join(" "); return candidate.docker_registry_image_name === process.env.WORKER_IMAGE && allowed.has(candidate.uuid) && (envPattern.test(text) || namePattern.test(text)); }); process.stdout.write(app?.uuid || "");' "$APPS_JSON"
}
@@ -340,19 +365,20 @@ jobs:
WORKER_WEBHOOK_RESOURCE_UUID="$(select_worker_resource webhook)"
WORKER_PLUGIN_RESOURCE_UUID="$(select_worker_resource plugin)"
WORKER_RESOURCE_UUIDS="$WORKER_GENERAL_RESOURCE_UUID,$WORKER_MAIL_RESOURCE_UUID,$WORKER_IMPORT_RESOURCE_UUID,$WORKER_WEBHOOK_RESOURCE_UUID,$WORKER_PLUGIN_RESOURCE_UUID"
if [ -z "$WEB_RESOURCE_UUID" ] || [ -z "$API_RESOURCE_UUID" ] || [ -z "$WORKER_GENERAL_RESOURCE_UUID" ] || [ -z "$WORKER_MAIL_RESOURCE_UUID" ] || [ -z "$WORKER_IMPORT_RESOURCE_UUID" ] || [ -z "$WORKER_WEBHOOK_RESOURCE_UUID" ] || [ -z "$WORKER_PLUGIN_RESOURCE_UUID" ]; then
echo "Coolify staging web, api, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications must all exist"
if [ -z "$WEB_RESOURCE_UUID" ] || [ -z "$API_RESOURCE_UUID" ] || [ -z "$MCP_RESOURCE_UUID" ] || [ -z "$WORKER_GENERAL_RESOURCE_UUID" ] || [ -z "$WORKER_MAIL_RESOURCE_UUID" ] || [ -z "$WORKER_IMPORT_RESOURCE_UUID" ] || [ -z "$WORKER_WEBHOOK_RESOURCE_UUID" ] || [ -z "$WORKER_PLUGIN_RESOURCE_UUID" ]; then
echo "Coolify staging web, api, mcp, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications must all exist"
exit 1
fi
for REQUIRED_RESOURCE_UUID in "$WEB_RESOURCE_UUID" "$API_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
for REQUIRED_RESOURCE_UUID in "$WEB_RESOURCE_UUID" "$API_RESOURCE_UUID" "$MCP_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
if ! printf ",%s," "$RESOLVED_COOLIFY_STAGING_RESOURCE_UUIDS" | grep -Fq ",$REQUIRED_RESOURCE_UUID,"; then
echo "Coolify staging deploy resources must include the web, api, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications"
echo "Coolify staging deploy resources must include the web, api, mcp, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications"
exit 1
fi
done
DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"
DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$MCP_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"
{
echo "STAGING_WEB_RESOURCE_UUID=$WEB_RESOURCE_UUID"
echo "STAGING_MCP_RESOURCE_UUID=$MCP_RESOURCE_UUID"
echo "STAGING_WORKER_RESOURCE_UUIDS=$WORKER_RESOURCE_UUIDS"
echo "STAGING_WORKER_GENERAL_RESOURCE_UUID=$WORKER_GENERAL_RESOURCE_UUID"
echo "STAGING_WORKER_MAIL_RESOURCE_UUID=$WORKER_MAIL_RESOURCE_UUID"
@@ -362,14 +388,19 @@ jobs:
echo "STAGING_DEPENDENT_RESOURCE_UUIDS=$DEPENDENT_RESOURCE_UUIDS"
} >> "$GITHUB_ENV"
MIGRATION_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ pre_deployment_command: process.env.MIGRATION_COMMAND, post_deployment_command: process.env.SEED_COMMAND, docker_registry_image_tag: process.env.IMAGE_TAG }))')"
MIGRATION_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ pre_deployment_command: "", post_deployment_command: "", docker_registry_image_tag: process.env.IMAGE_TAG }))')"
curl --fail --silent --show-error --request PATCH \
--header "Authorization: Bearer $SELECTED_COOLIFY_TOKEN" \
--header "Content-Type: application/json" \
--data "$MIGRATION_PAYLOAD" \
"$SELECTED_COOLIFY_API_URL/applications/$WEB_RESOURCE_UUID" >/dev/null
node scripts/coolify-deploy.mjs set-envs "$WEB_RESOURCE_UUID" \
MIGRATE_ON_BOOT=true \
SEED_ON_BOOT=true \
SEED_DEMO_DATA=true \
ALLOW_STAGING_DEMO_SEED=true
IMAGE_TAG_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ docker_registry_image_tag: process.env.IMAGE_TAG }))')"
for RESOURCE_UUID in "$API_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
for RESOURCE_UUID in "$API_RESOURCE_UUID" "$MCP_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
curl --fail --silent --show-error --request PATCH \
--header "Authorization: Bearer $SELECTED_COOLIFY_TOKEN" \
--header "Content-Type: application/json" \
@@ -384,26 +415,11 @@ jobs:
exit 0
fi
for TARGET in web api worker; do
for TARGET in web api worker mcp; do
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
docker buildx imagetools create --tag "$IMAGE:staging" "$IMAGE:$IMAGE_TAG"
done
- name: Pre-pull staging images on Coolify host
env:
COOLIFY_STAGING_SSH_HOST: ${{ vars.COOLIFY_STAGING_SSH_HOST }}
COOLIFY_STAGING_SSH_USER: ${{ vars.COOLIFY_STAGING_SSH_USER }}
COOLIFY_STAGING_SSH_KEY: ${{ secrets.COOLIFY_STAGING_SSH_KEY }}
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
REGISTRY_PAT: ${{ secrets.REGISTRY_PAT }}
run: |
if [ "$DEPLOY_ACTION" != "stage" ]; then
echo "Staging image pre-pull skipped for promote"
exit 0
fi
timeout 240s scripts/coolify-prepull-images.sh || echo "Staging image pre-pull skipped after timeout; Coolify will pull during deploy"
- name: Trigger Coolify staging deploy
env:
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
@@ -457,7 +473,7 @@ jobs:
EXPECTED_IMAGE_ALIAS="${4:-staging}"
EXPECTED_SERVICE="${5:-}"
HEALTH_URL="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const raw = app.fqdn || app.domain || app.domains || ""; const value = Array.isArray(raw) ? raw[0] : String(raw).split(/[,\s]+/).find(Boolean); process.stdout.write(value ? value.replace(/\/+$/, "") : "");' "$APP_JSON")"
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api|mcp)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
if [ -z "$HEALTH_URL" ]; then
if [ "$REQUIRE_HEALTH_URL" = "required" ] || [ "$HTTP_RESOURCE" = "true" ]; then
echo "$LABEL is an HTTP resource but has no health URL"
@@ -555,9 +571,6 @@ jobs:
env:
COOLIFY_STAGING_API_URL: ${{ vars.COOLIFY_STAGING_API_URL }}
COOLIFY_STAGING_TOKEN: ${{ secrets.COOLIFY_STAGING_TOKEN }}
COOLIFY_STAGING_SSH_HOST: ${{ vars.COOLIFY_STAGING_SSH_HOST }}
COOLIFY_STAGING_SSH_USER: ${{ vars.COOLIFY_STAGING_SSH_USER }}
COOLIFY_STAGING_SSH_KEY: ${{ secrets.COOLIFY_STAGING_SSH_KEY }}
run: |
if [ "$DEPLOY_ACTION" != "stage" ]; then
echo "Staging dependent health wait skipped for promote"
@@ -580,35 +593,6 @@ jobs:
exit 1
fi
# SSH helper for worker health verification (Coolify has no exec REST API);
# used by verify_live_revision to run the health probe via docker exec.
if [ -z "$COOLIFY_STAGING_SSH_HOST" ] || [ -z "$COOLIFY_STAGING_SSH_KEY" ]; then
echo "Coolify staging SSH configuration is incomplete (worker health verification requires SSH + docker exec)"
exit 1
fi
REMOTE_SSH_KEY_FILE="$(mktemp)"
REMOTE_SSH_KNOWN_HOSTS="$(mktemp)"
trap 'rm -f "$REMOTE_SSH_KEY_FILE" "$REMOTE_SSH_KNOWN_HOSTS"' EXIT
# Materialise the key via an intermediate var so the secret is written to
# a file (never stdout/logs); avoids leaking it in CI output.
REMOTE_SSH_KEY_DATA="$COOLIFY_STAGING_SSH_KEY"
printf '%s\n' "$REMOTE_SSH_KEY_DATA" > "$REMOTE_SSH_KEY_FILE"
unset REMOTE_SSH_KEY_DATA
chmod 600 "$REMOTE_SSH_KEY_FILE"
REMOTE_SSH_USER="${COOLIFY_STAGING_SSH_USER:-root}"
REMOTE_SSH_HOST="$COOLIFY_STAGING_SSH_HOST"
if command -v ssh-keyscan >/dev/null 2>&1; then
ssh-keyscan -H "$REMOTE_SSH_HOST" > "$REMOTE_SSH_KNOWN_HOSTS" 2>/dev/null || true
fi
if [ -s "$REMOTE_SSH_KNOWN_HOSTS" ]; then
REMOTE_SSH_HOSTKEY_OPT="-o StrictHostKeyChecking=yes"
else
REMOTE_SSH_HOSTKEY_OPT="-o StrictHostKeyChecking=accept-new"
fi
remote_ssh() {
ssh -i "$REMOTE_SSH_KEY_FILE" -o BatchMode=yes -o ConnectTimeout=10 -o UserKnownHostsFile="$REMOTE_SSH_KNOWN_HOSTS" $REMOTE_SSH_HOSTKEY_OPT "${REMOTE_SSH_USER}@${REMOTE_SSH_HOST}" "$@"
}
DEPLOYMENT_INDEX=0
DEPLOYMENT_WAIT_PIDS=""
for DEPLOYMENT_UUID in $(printf "%s" "$STAGING_DEPENDENT_DEPLOYMENT_UUIDS" | tr ',' ' '); do
@@ -636,7 +620,7 @@ jobs:
RESOURCE_KIND="${6:-auto}"
EXPECTED_WORKER_GROUP="${7:-}"
HEALTH_URL="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const raw = app.fqdn || app.domain || app.domains || ""; const value = Array.isArray(raw) ? raw[0] : String(raw).split(/[,\s]+/).find(Boolean); process.stdout.write(value ? value.replace(/\/+$/, "") : "");' "$APP_JSON")"
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api|mcp)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
WORKER_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])worker(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
if [ "$RESOURCE_KIND" = "worker" ] || [ "$WORKER_RESOURCE" = "true" ]; then
CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")"
@@ -644,74 +628,7 @@ jobs:
echo "$LABEL worker image tag does not match expected alias"
exit 1
fi
# Coolify 4.x exposes no command-execution REST API, so verify worker
# health by running the one-shot health probe inside the live container
# over SSH (docker exec) on the Coolify host. Fail-closed: any SSH /
# container-resolution / exec failure blocks the deploy.
WORKER_CONTAINER="$(remote_ssh "docker ps --filter name=$RESOURCE_UUID --filter status=running --format '{{.Names}}' | head -n1" 2>/dev/null | tr -d '\r')"
if [ -z "$WORKER_CONTAINER" ]; then
echo "$LABEL worker container not found on Coolify host"
exit 1
fi
# health.js exits non-zero when the worker reports a non-ready
# status (e.g. "misconfigured"), yet it still prints the diagnostic
# JSON (status + errors) to stdout. Capture stdout regardless of the
# exit code (|| true) so the real reason surfaces in the checks below
# instead of a generic "command unavailable". Empty output means the
# docker exec itself could not run (container gone, SSH failure, ...).
WORKER_HEALTH_OUTPUT="$(remote_ssh "docker exec $WORKER_CONTAINER node apps/worker/dist/health.js" 2>/dev/null | tr -d '\r' || true)"
if [ -z "$WORKER_HEALTH_OUTPUT" ]; then
echo "$LABEL worker docker exec health command unavailable (no output from health probe)"
exit 1
fi
WORKER_HEALTH_JSON="$(mktemp)"
printf "%s" "$WORKER_HEALTH_OUTPUT" > "$WORKER_HEALTH_JSON"
if ! WORKER_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$WORKER_HEALTH_JSON" 2>/dev/null)"; then
rm -f "$WORKER_HEALTH_JSON"
echo "$LABEL worker execute output is not JSON"
exit 1
fi
WORKER_STATUS="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.status || "");' "$WORKER_HEALTH_JSON")"
WORKER_ENABLED="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.workerEnabled === true ? "true" : "false");' "$WORKER_HEALTH_JSON")"
WORKER_GROUP="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.workerGroup || "");' "$WORKER_HEALTH_JSON")"
WORKER_QUEUES="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(Array.isArray(payload.queues) ? payload.queues.join(",") : "");' "$WORKER_HEALTH_JSON")"
WORKER_HEALTH_ERRORS="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(Array.isArray(payload.errors) ? payload.errors.join("; ") : "");' "$WORKER_HEALTH_JSON")"
rm -f "$WORKER_HEALTH_JSON"
if [ "$WORKER_REVISION" != "$IMAGE_TAG" ]; then
echo "$LABEL worker health revision does not match image tag yet"
return 1
fi
if [ "$WORKER_ENABLED" != "true" ]; then
echo "$LABEL worker health reports workerEnabled=false"
exit 1
fi
if [ "$WORKER_STATUS" != "ready" ]; then
echo "$LABEL worker health status is not ready: '$WORKER_STATUS'${WORKER_HEALTH_ERRORS:+ (errors: $WORKER_HEALTH_ERRORS)}"
exit 1
fi
if [ -n "$EXPECTED_WORKER_GROUP" ] && [ "$WORKER_GROUP" != "$EXPECTED_WORKER_GROUP" ]; then
echo "$LABEL worker group does not match expected queue group"
exit 1
fi
EXPECTED_WORKER_QUEUES=""
case "$EXPECTED_WORKER_GROUP" in
general) EXPECTED_WORKER_QUEUES="toolctrl.notifications toolctrl.maintenance toolctrl.file-scan" ;;
mail) EXPECTED_WORKER_QUEUES="toolctrl.email" ;;
import)
if [ -n "$WORKER_QUEUES" ]; then
EXPECTED_WORKER_QUEUES="toolctrl.imports toolctrl.exports"
fi
;;
webhook) EXPECTED_WORKER_QUEUES="toolctrl.ai-chatbot toolctrl.browser-automation" ;;
plugin) EXPECTED_WORKER_QUEUES="toolctrl.plugin-invoke" ;;
esac
for EXPECTED_WORKER_QUEUE in $EXPECTED_WORKER_QUEUES; do
if ! printf ",%s," "$WORKER_QUEUES" | grep -Fq ",$EXPECTED_WORKER_QUEUE,"; then
echo "$LABEL worker queues do not match expected queue group"
exit 1
fi
done
echo "$LABEL worker live revision matches image tag"
echo "$LABEL worker Coolify status is healthy and image tag matches expected alias"
return
fi
if [ -z "$HEALTH_URL" ]; then
@@ -761,6 +678,10 @@ jobs:
RESOURCE_KIND="api"
EXPECTED_SERVICE="api"
EXPECTED_WORKER_GROUP=""
if [ "$RESOURCE_UUID" = "$STAGING_MCP_RESOURCE_UUID" ]; then
RESOURCE_KIND="mcp"
EXPECTED_SERVICE="mcp"
fi
if printf ",%s," "$STAGING_WORKER_RESOURCE_UUIDS" | grep -Fq ",$RESOURCE_UUID,"; then
RESOURCE_KIND="worker"
EXPECTED_SERVICE=""
@@ -829,16 +750,17 @@ jobs:
node scripts/coolify-ensure-worker-groups.mjs production
- name: Configure Coolify production migration command
# Same entrypoint-based migration mechanism as staging: boot flags on the
# web resource, dead Coolify hook slots cleared. Production runs the
# ungated seed (no demo flags) inside the new container after migrate.
- name: Configure Coolify production migration boot flags
env:
COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}
COOLIFY_RESOURCE_UUIDS: ${{ vars.COOLIFY_RESOURCE_UUIDS }}
COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }}
MIGRATION_COMMAND: cd /myapp && pnpm exec prisma migrate deploy --config prisma.config.ts
SEED_COMMAND: cd /myapp && NODE_ENV=production node packages/db/dist/production-seed.server.js
run: |
if [ "$DEPLOY_ACTION" != "promote" ]; then
echo "Coolify production migration command configuration skipped for staging"
echo "Coolify production migration boot flag configuration skipped for staging"
exit 0
fi
@@ -850,6 +772,7 @@ jobs:
WEB_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/web"
API_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/api"
MCP_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/mcp"
WORKER_IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/worker"
APPS_JSON="$(mktemp)"
trap 'rm -f "${APPS_JSON:-}"' EXIT
@@ -858,6 +781,7 @@ jobs:
"$COOLIFY_API_URL/applications" > "$APPS_JSON"
WEB_RESOURCE_UUID="$(WEB_IMAGE="$WEB_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.WEB_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
API_RESOURCE_UUID="$(API_IMAGE="$API_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.API_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
MCP_RESOURCE_UUID="$(MCP_IMAGE="$MCP_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const app = apps.find((candidate) => candidate.docker_registry_image_name === process.env.MCP_IMAGE && allowed.has(candidate.uuid)); process.stdout.write(app?.uuid || "");' "$APPS_JSON")"
select_worker_resource() {
WORKER_QUEUE_GROUP="$1" WORKER_IMAGE="$WORKER_IMAGE" ALLOWED_RESOURCE_UUIDS="$RESOLVED_COOLIFY_RESOURCE_UUIDS" node -e 'const fs = require("node:fs"); const apps = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const allowed = new Set((process.env.ALLOWED_RESOURCE_UUIDS || "").split(",").filter(Boolean)); const group = process.env.WORKER_QUEUE_GROUP || ""; const envPattern = new RegExp(`WORKER_QUEUE_GROUP\\s*(?:=|:)\\s*["'\''"]?${group}\\b`, "i"); const namePattern = new RegExp(`(?:^|[\\s/_-])worker[\\s_-]*${group}(?:$|[\\s:_-])`, "i"); const app = apps.find((candidate) => { const text = [candidate.name, candidate.description, candidate.docker_registry_image_name, candidate.fqdn, candidate.domain, candidate.domains, JSON.stringify(candidate.environment_variables || candidate.envs || candidate.environment || "")].filter(Boolean).join(" "); return candidate.docker_registry_image_name === process.env.WORKER_IMAGE && allowed.has(candidate.uuid) && (envPattern.test(text) || namePattern.test(text)); }); process.stdout.write(app?.uuid || "");' "$APPS_JSON"
}
@@ -867,19 +791,20 @@ jobs:
WORKER_WEBHOOK_RESOURCE_UUID="$(select_worker_resource webhook)"
WORKER_PLUGIN_RESOURCE_UUID="$(select_worker_resource plugin)"
WORKER_RESOURCE_UUIDS="$WORKER_GENERAL_RESOURCE_UUID,$WORKER_MAIL_RESOURCE_UUID,$WORKER_IMPORT_RESOURCE_UUID,$WORKER_WEBHOOK_RESOURCE_UUID,$WORKER_PLUGIN_RESOURCE_UUID"
if [ -z "$WEB_RESOURCE_UUID" ] || [ -z "$API_RESOURCE_UUID" ] || [ -z "$WORKER_GENERAL_RESOURCE_UUID" ] || [ -z "$WORKER_MAIL_RESOURCE_UUID" ] || [ -z "$WORKER_IMPORT_RESOURCE_UUID" ] || [ -z "$WORKER_WEBHOOK_RESOURCE_UUID" ] || [ -z "$WORKER_PLUGIN_RESOURCE_UUID" ]; then
echo "Coolify production web, api, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications must all exist"
if [ -z "$WEB_RESOURCE_UUID" ] || [ -z "$API_RESOURCE_UUID" ] || [ -z "$MCP_RESOURCE_UUID" ] || [ -z "$WORKER_GENERAL_RESOURCE_UUID" ] || [ -z "$WORKER_MAIL_RESOURCE_UUID" ] || [ -z "$WORKER_IMPORT_RESOURCE_UUID" ] || [ -z "$WORKER_WEBHOOK_RESOURCE_UUID" ] || [ -z "$WORKER_PLUGIN_RESOURCE_UUID" ]; then
echo "Coolify production web, api, mcp, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications must all exist"
exit 1
fi
for REQUIRED_RESOURCE_UUID in "$WEB_RESOURCE_UUID" "$API_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
for REQUIRED_RESOURCE_UUID in "$WEB_RESOURCE_UUID" "$API_RESOURCE_UUID" "$MCP_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
if ! printf ",%s," "$RESOLVED_COOLIFY_RESOURCE_UUIDS" | grep -Fq ",$REQUIRED_RESOURCE_UUID,"; then
echo "Coolify production deploy resources must include the web, api, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications"
echo "Coolify production deploy resources must include the web, api, mcp, worker-general, worker-mail, worker-import, worker-webhook, and worker-plugin applications"
exit 1
fi
done
DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"
DEPENDENT_RESOURCE_UUIDS="$API_RESOURCE_UUID,$MCP_RESOURCE_UUID,$WORKER_RESOURCE_UUIDS"
{
echo "PRODUCTION_WEB_RESOURCE_UUID=$WEB_RESOURCE_UUID"
echo "PRODUCTION_MCP_RESOURCE_UUID=$MCP_RESOURCE_UUID"
echo "PRODUCTION_WORKER_RESOURCE_UUIDS=$WORKER_RESOURCE_UUIDS"
echo "PRODUCTION_WORKER_GENERAL_RESOURCE_UUID=$WORKER_GENERAL_RESOURCE_UUID"
echo "PRODUCTION_WORKER_MAIL_RESOURCE_UUID=$WORKER_MAIL_RESOURCE_UUID"
@@ -889,14 +814,17 @@ jobs:
echo "PRODUCTION_DEPENDENT_RESOURCE_UUIDS=$DEPENDENT_RESOURCE_UUIDS"
} >> "$GITHUB_ENV"
MIGRATION_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ pre_deployment_command: process.env.MIGRATION_COMMAND, post_deployment_command: process.env.SEED_COMMAND, docker_registry_image_tag: process.env.IMAGE_TAG }))')"
MIGRATION_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ pre_deployment_command: "", post_deployment_command: "", docker_registry_image_tag: process.env.IMAGE_TAG }))')"
curl --fail --silent --show-error --request PATCH \
--header "Authorization: Bearer $COOLIFY_TOKEN" \
--header "Content-Type: application/json" \
--data "$MIGRATION_PAYLOAD" \
"$COOLIFY_API_URL/applications/$WEB_RESOURCE_UUID" >/dev/null
node scripts/coolify-deploy.mjs set-envs "$WEB_RESOURCE_UUID" \
MIGRATE_ON_BOOT=true \
SEED_ON_BOOT=true
IMAGE_TAG_PAYLOAD="$(node -e 'process.stdout.write(JSON.stringify({ docker_registry_image_tag: process.env.IMAGE_TAG }))')"
for RESOURCE_UUID in "$API_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
for RESOURCE_UUID in "$API_RESOURCE_UUID" "$MCP_RESOURCE_UUID" "$WORKER_GENERAL_RESOURCE_UUID" "$WORKER_MAIL_RESOURCE_UUID" "$WORKER_IMPORT_RESOURCE_UUID" "$WORKER_WEBHOOK_RESOURCE_UUID" "$WORKER_PLUGIN_RESOURCE_UUID"; do
curl --fail --silent --show-error --request PATCH \
--header "Authorization: Bearer $COOLIFY_TOKEN" \
--header "Content-Type: application/json" \
@@ -911,7 +839,7 @@ jobs:
exit 0
fi
for TARGET in web api worker; do
for TARGET in web api worker mcp; do
IMAGE="$REGISTRY/$IMAGE_NAMESPACE/$IMAGE_PREFIX/$TARGET"
docker buildx imagetools create --tag "$IMAGE:main" "$IMAGE:$IMAGE_TAG"
done
@@ -968,7 +896,7 @@ jobs:
EXPECTED_IMAGE_ALIAS="${4:-main}"
EXPECTED_SERVICE="${5:-}"
HEALTH_URL="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const raw = app.fqdn || app.domain || app.domains || ""; const value = Array.isArray(raw) ? raw[0] : String(raw).split(/[,\s]+/).find(Boolean); process.stdout.write(value ? value.replace(/\/+$/, "") : "");' "$APP_JSON")"
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api|mcp)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
if [ -z "$HEALTH_URL" ]; then
if [ "$REQUIRE_HEALTH_URL" = "required" ] || [ "$HTTP_RESOURCE" = "true" ]; then
echo "$LABEL is an HTTP resource but has no health URL"
@@ -1118,12 +1046,6 @@ jobs:
env:
COOLIFY_API_URL: ${{ vars.COOLIFY_API_URL }}
COOLIFY_TOKEN: ${{ secrets.COOLIFY_TOKEN }}
# AD-24: Prod ist Multi-Server. Worker laufen auf der worker-VM (privates Netz), nur ueber
# die control-Bastion erreichbar -> Worker-Health-Verifikation (docker exec) per SSH-ProxyJump.
COOLIFY_PROD_SSH_KEY: ${{ secrets.COOLIFY_PROD_SSH_KEY }}
COOLIFY_PROD_SSH_KNOWN_HOSTS: ${{ secrets.COOLIFY_PROD_SSH_KNOWN_HOSTS }}
COOLIFY_PROD_SSH_HOST: ${{ vars.COOLIFY_PROD_SSH_HOST }}
COOLIFY_PROD_WORKER_PRIVATE_IP: ${{ vars.COOLIFY_PROD_WORKER_PRIVATE_IP }}
run: |
if [ "$DEPLOY_ACTION" != "promote" ]; then
echo "Production dependent health wait skipped for staging"
@@ -1164,32 +1086,6 @@ jobs:
exit 1
fi
# SSH helper for worker health verification (Coolify has no exec REST API). AD-24: Prod ist
# Multi-Server -> Worker laufen auf der worker-VM (privates Netz), erreichbar nur ueber die
# control-Bastion. ProxyJump + gepinnte Host-Keys (kein StrictHostKeyChecking=no). Secrets
# werden via Zwischen-Variable in Dateien geschrieben (nie stdout/Logs).
if [ -z "$COOLIFY_PROD_SSH_HOST" ] || [ -z "$COOLIFY_PROD_SSH_KEY" ] || [ -z "$COOLIFY_PROD_SSH_KNOWN_HOSTS" ] || [ -z "$COOLIFY_PROD_WORKER_PRIVATE_IP" ]; then
echo "Coolify production SSH configuration is incomplete (worker health verification requires SSH + docker exec)"
exit 1
fi
REMOTE_SSH_KEY_FILE="$(mktemp)"
REMOTE_SSH_KNOWN_HOSTS="$(mktemp)"
trap 'rm -f "$REMOTE_SSH_KEY_FILE" "$REMOTE_SSH_KNOWN_HOSTS"' EXIT
REMOTE_SSH_KEY_DATA="$COOLIFY_PROD_SSH_KEY"
printf '%s\n' "$REMOTE_SSH_KEY_DATA" > "$REMOTE_SSH_KEY_FILE"
unset REMOTE_SSH_KEY_DATA
chmod 600 "$REMOTE_SSH_KEY_FILE"
REMOTE_SSH_KNOWN_HOSTS_DATA="$COOLIFY_PROD_SSH_KNOWN_HOSTS"
printf '%s\n' "$REMOTE_SSH_KNOWN_HOSTS_DATA" > "$REMOTE_SSH_KNOWN_HOSTS"
unset REMOTE_SSH_KNOWN_HOSTS_DATA
# ProxyCommand (nicht ProxyJump): ProxyJump reicht den -i-Key nicht zuverlaessig an den
# Jump-Hop weiter (MaxAuthTries -> Permission denied). ProxyCommand gibt der Bastion den Key
# explizit (IdentitiesOnly), Host-Keys fuer beide Hops gepinnt.
REMOTE_SSH_PROXY="ssh -i $REMOTE_SSH_KEY_FILE -o IdentitiesOnly=yes -o BatchMode=yes -o ConnectTimeout=20 -o UserKnownHostsFile=$REMOTE_SSH_KNOWN_HOSTS -o StrictHostKeyChecking=yes -W %h:%p root@${COOLIFY_PROD_SSH_HOST}"
remote_ssh() {
ssh -i "$REMOTE_SSH_KEY_FILE" -o IdentitiesOnly=yes -o BatchMode=yes -o ConnectTimeout=20 -o UserKnownHostsFile="$REMOTE_SSH_KNOWN_HOSTS" -o StrictHostKeyChecking=yes -o ProxyCommand="$REMOTE_SSH_PROXY" "root@${COOLIFY_PROD_WORKER_PRIVATE_IP}" "$@"
}
verify_live_revision() {
APP_JSON="$1"
LABEL="$2"
@@ -1199,7 +1095,7 @@ jobs:
RESOURCE_KIND="${6:-auto}"
EXPECTED_WORKER_GROUP="${7:-}"
HEALTH_URL="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const raw = app.fqdn || app.domain || app.domains || ""; const value = Array.isArray(raw) ? raw[0] : String(raw).split(/[,\s]+/).find(Boolean); process.stdout.write(value ? value.replace(/\/+$/, "") : "");' "$APP_JSON")"
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
HTTP_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])(?:web|api|mcp)(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
WORKER_RESOURCE="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); const value = [app.docker_registry_image_name, app.name, app.description].filter(Boolean).join(" "); process.stdout.write(/(?:^|[\\/\\s_-])worker(?:$|[\\s:_-])/i.test(value) ? "true" : "false");' "$APP_JSON")"
if [ "$RESOURCE_KIND" = "worker" ] || [ "$WORKER_RESOURCE" = "true" ]; then
CONFIGURED_IMAGE_TAG="$(node -e 'const fs = require("node:fs"); const app = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(app.docker_registry_image_tag || app.image_tag || app.tag || "");' "$APP_JSON")"
@@ -1207,74 +1103,7 @@ jobs:
echo "$LABEL worker image tag does not match expected alias"
exit 1
fi
# Coolify 4.x exposes no command-execution REST API, so verify worker
# health by running the one-shot health probe inside the live container
# over SSH (docker exec) on the Coolify host. Fail-closed: any SSH /
# container-resolution / exec failure blocks the deploy.
WORKER_CONTAINER="$(remote_ssh "docker ps --filter name=$RESOURCE_UUID --filter status=running --format '{{.Names}}' | head -n1" 2>/dev/null | tr -d '\r')"
if [ -z "$WORKER_CONTAINER" ]; then
echo "$LABEL worker container not found on Coolify host"
exit 1
fi
# health.js exits non-zero when the worker reports a non-ready
# status (e.g. "misconfigured"), yet it still prints the diagnostic
# JSON (status + errors) to stdout. Capture stdout regardless of the
# exit code (|| true) so the real reason surfaces in the checks below
# instead of a generic "command unavailable". Empty output means the
# docker exec itself could not run (container gone, SSH failure, ...).
WORKER_HEALTH_OUTPUT="$(remote_ssh "docker exec $WORKER_CONTAINER node apps/worker/dist/health.js" 2>/dev/null | tr -d '\r' || true)"
if [ -z "$WORKER_HEALTH_OUTPUT" ]; then
echo "$LABEL worker docker exec health command unavailable (no output from health probe)"
exit 1
fi
WORKER_HEALTH_JSON="$(mktemp)"
printf "%s" "$WORKER_HEALTH_OUTPUT" > "$WORKER_HEALTH_JSON"
if ! WORKER_REVISION="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.revision || "");' "$WORKER_HEALTH_JSON" 2>/dev/null)"; then
rm -f "$WORKER_HEALTH_JSON"
echo "$LABEL worker execute output is not JSON"
exit 1
fi
WORKER_STATUS="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.status || "");' "$WORKER_HEALTH_JSON")"
WORKER_ENABLED="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.workerEnabled === true ? "true" : "false");' "$WORKER_HEALTH_JSON")"
WORKER_GROUP="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(payload.workerGroup || "");' "$WORKER_HEALTH_JSON")"
WORKER_QUEUES="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(Array.isArray(payload.queues) ? payload.queues.join(",") : "");' "$WORKER_HEALTH_JSON")"
WORKER_HEALTH_ERRORS="$(node -e 'const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8")); process.stdout.write(Array.isArray(payload.errors) ? payload.errors.join("; ") : "");' "$WORKER_HEALTH_JSON")"
rm -f "$WORKER_HEALTH_JSON"
if [ "$WORKER_REVISION" != "$IMAGE_TAG" ]; then
echo "$LABEL worker health revision does not match image tag yet"
return 1
fi
if [ "$WORKER_ENABLED" != "true" ]; then
echo "$LABEL worker health reports workerEnabled=false"
exit 1
fi
if [ "$WORKER_STATUS" != "ready" ]; then
echo "$LABEL worker health status is not ready: '$WORKER_STATUS'${WORKER_HEALTH_ERRORS:+ (errors: $WORKER_HEALTH_ERRORS)}"
exit 1
fi
if [ -n "$EXPECTED_WORKER_GROUP" ] && [ "$WORKER_GROUP" != "$EXPECTED_WORKER_GROUP" ]; then
echo "$LABEL worker group does not match expected queue group"
exit 1
fi
EXPECTED_WORKER_QUEUES=""
case "$EXPECTED_WORKER_GROUP" in
general) EXPECTED_WORKER_QUEUES="toolctrl.notifications toolctrl.maintenance toolctrl.file-scan" ;;
mail) EXPECTED_WORKER_QUEUES="toolctrl.email" ;;
import)
if [ -n "$WORKER_QUEUES" ]; then
EXPECTED_WORKER_QUEUES="toolctrl.imports toolctrl.exports"
fi
;;
webhook) EXPECTED_WORKER_QUEUES="toolctrl.ai-chatbot toolctrl.browser-automation" ;;
plugin) EXPECTED_WORKER_QUEUES="toolctrl.plugin-invoke" ;;
esac
for EXPECTED_WORKER_QUEUE in $EXPECTED_WORKER_QUEUES; do
if ! printf ",%s," "$WORKER_QUEUES" | grep -Fq ",$EXPECTED_WORKER_QUEUE,"; then
echo "$LABEL worker queues do not match expected queue group"
exit 1
fi
done
echo "$LABEL worker live revision matches image tag"
echo "$LABEL worker Coolify status is healthy and image tag matches expected alias"
return
fi
if [ -z "$HEALTH_URL" ]; then
@@ -1324,6 +1153,10 @@ jobs:
RESOURCE_KIND="api"
EXPECTED_SERVICE="api"
EXPECTED_WORKER_GROUP=""
if [ "$RESOURCE_UUID" = "$PRODUCTION_MCP_RESOURCE_UUID" ]; then
RESOURCE_KIND="mcp"
EXPECTED_SERVICE="mcp"
fi
if printf ",%s," "$PRODUCTION_WORKER_RESOURCE_UUIDS" | grep -Fq ",$RESOURCE_UUID,"; then
RESOURCE_KIND="worker"
EXPECTED_SERVICE=""
@@ -1390,7 +1223,7 @@ jobs:
SENTRY_ENVIRONMENT=staging
RELEASE_SHA="${SHA}"
fi
for PROJECT in boilerplate-api boilerplate-worker boilerplate-web; do
for PROJECT in api worker web; do
bash scripts/ci/sentry-release.sh \
--project "$PROJECT" \
--environment "$SENTRY_ENVIRONMENT" \